ingestr 0.12.9__py3-none-any.whl → 0.12.10__py3-none-any.whl

ingestr/src/factory.py

@@ -33,6 +33,7 @@ from ingestr.src.sources import (
     HubspotSource,
     KafkaSource,
     KlaviyoSource,
+    LinkedInAdsSource,
     LocalCsvSource,
     MongoDbSource,
     NotionSource,
@@ -126,6 +127,7 @@ class SourceDestinationFactory:
         "googleanalytics": GoogleAnalyticsSource,
         "appstore": AppleAppStoreSource,
         "gs": GCSSource,
+        "linkedinads": LinkedInAdsSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/linkedin_ads/__init__.py

@@ -0,0 +1,63 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from pendulum import Date
+
+from .dimension_time_enum import Dimension, TimeGranularity
+from .helpers import LinkedInAdsAPI, find_intervals
+
+
+@dlt.source(max_table_nesting=0)
+def linked_in_ads_source(
+    start_date: Date,
+    end_date: Date | None,
+    access_token: str,
+    account_ids: list[str],
+    dimension: Dimension,
+    metrics: list[str],
+    time_granularity: TimeGranularity,
+) -> DltResource:
+    linkedin_api = LinkedInAdsAPI(
+        access_token=access_token,
+        account_ids=account_ids,
+        dimension=dimension,
+        metrics=metrics,
+        time_granularity=time_granularity,
+    )
+
+    if time_granularity == TimeGranularity.daily:
+        primary_key = [dimension.value, "date"]
+        incremental_loading_param = "date"
+    else:
+        primary_key = [dimension.value, "start_date", "end_date"]
+        incremental_loading_param = "start_date"
+
+    @dlt.resource(write_disposition="merge", primary_key=primary_key)
+    def custom_reports(
+        dateTime=(
+            dlt.sources.incremental(
+                incremental_loading_param,
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterable[TDataItem]:
+        if dateTime.end_value is None:
+            end_date = pendulum.now().date()
+        else:
+            end_date = dateTime.end_value
+
+        list_of_interval = find_intervals(
+            start_date=dateTime.last_value,
+            end_date=end_date,
+            time_granularity=time_granularity,
+        )
+        for start, end in list_of_interval:
+            yield linkedin_api.fetch_pages(start, end)
+
+    return custom_reports

ingestr/src/linkedin_ads/dimension_time_enum.py

@@ -0,0 +1,12 @@
+from enum import Enum
+
+
+class Dimension(Enum):
+    campaign = "campaign"
+    creative = "creative"
+    account = "account"
+
+
+class TimeGranularity(Enum):
+    daily = "DAILY"
+    monthly = "MONTHLY"

ingestr/src/linkedin_ads/helpers.py

@@ -0,0 +1,148 @@
+from urllib.parse import quote
+
+import pendulum
+import requests
+from dlt.sources.helpers.requests import Client
+from pendulum import Date
+
+from .dimension_time_enum import Dimension, TimeGranularity
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 429
+
+
+def create_client() -> requests.Session:
+    return Client(
+        request_timeout=10.0,
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+    ).session
+
+
+def flat_structure(items, pivot: Dimension, time_granularity: TimeGranularity):
+    for item in items:
+        if "pivotValues" in item:
+            if len(item["pivotValues"]) > 1:
+                item[pivot.value.lower()] = item["pivotValues"]
+            else:
+                item[pivot.value.lower()] = item["pivotValues"][0]
+        if "dateRange" in item:
+            start_date = item["dateRange"]["start"]
+            start_dt = pendulum.date(
+                year=start_date["year"],
+                month=start_date["month"],
+                day=start_date["day"],
+            )
+            if time_granularity == TimeGranularity.daily:
+                item["date"] = start_dt
+            else:
+                end_date = item["dateRange"]["end"]
+                end_dt = pendulum.date(
+                    year=end_date["year"],
+                    month=end_date["month"],
+                    day=end_date["day"],
+                )
+                item["start_date"] = start_dt
+                item["end_date"] = end_dt
+
+        del item["dateRange"]
+        del item["pivotValues"]
+
+    return items
+
+
+def find_intervals(start_date: Date, end_date: Date, time_granularity: TimeGranularity):
+    intervals = []
+
+    if start_date > end_date:
+        raise ValueError("Start date must be less than end date")
+
+    while start_date <= end_date:
+        if time_granularity == TimeGranularity.daily:
+            next_date = min(start_date.add(months=6), end_date)
+        else:
+            next_date = min(start_date.add(years=2), end_date)
+
+        intervals.append((start_date, next_date))
+
+        start_date = next_date.add(days=1)
+
+    return intervals
+
+
+def construct_url(
+    start: Date,
+    end: Date,
+    account_ids: list[str],
+    metrics: list[str],
+    dimension: Dimension,
+    time_granularity: TimeGranularity,
+):
+    date_range = f"(start:(year:{start.year},month:{start.month},day:{start.day})"
+    date_range += f",end:(year:{end.year},month:{end.month},day:{end.day}))"
+    accounts = ",".join(
+        [quote(f"urn:li:sponsoredAccount:{account_id}") for account_id in account_ids]
+    )
+    encoded_accounts = f"List({accounts})"
+    dimension_str = dimension.value.upper()
+    time_granularity_str = time_granularity.value
+    metrics_str = ",".join([metric for metric in metrics])
+
+    url = (
+        f"https://api.linkedin.com/rest/adAnalytics?"
+        f"q=analytics&timeGranularity={time_granularity_str}&"
+        f"dateRange={date_range}&accounts={encoded_accounts}&"
+        f"pivot={dimension_str}&fields={metrics_str}"
+    )
+
+    return url
+
+
+class LinkedInAdsAPI:
+    def __init__(
+        self,
+        access_token,
+        time_granularity,
+        account_ids,
+        dimension,
+        metrics,
+    ):
+        self.time_granularity: TimeGranularity = time_granularity
+        self.account_ids: list[str] = account_ids
+        self.dimension: Dimension = dimension
+        self.metrics: list[str] = metrics
+        self.headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Linkedin-Version": "202411",
+            "X-Restli-Protocol-Version": "2.0.0",
+        }
+
+    def fetch_pages(self, start: Date, end: Date):
+        client = create_client()
+        url = construct_url(
+            start=start,
+            end=end,
+            account_ids=self.account_ids,
+            metrics=self.metrics,
+            dimension=self.dimension,
+            time_granularity=self.time_granularity,
+        )
+        response = client.get(url=url, headers=self.headers)
+
+        if response.status_code != 200:
+            error_data = response.json()
+            raise ValueError(f"LinkedIn API Error: {error_data.get('message')}")
+
+        result = response.json()
+        items = result.get("elements", [])
+        yield flat_structure(
+            items=items,
+            pivot=self.dimension,
+            time_granularity=self.time_granularity,
+        )

ingestr/src/sources.py

@@ -69,6 +69,11 @@ from ingestr.src.hubspot import hubspot
 from ingestr.src.kafka import kafka_consumer
 from ingestr.src.kafka.helpers import KafkaCredentials
 from ingestr.src.klaviyo._init_ import klaviyo_source
+from ingestr.src.linkedin_ads import linked_in_ads_source
+from ingestr.src.linkedin_ads.dimension_time_enum import (
+    Dimension,
+    TimeGranularity,
+)
 from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
 from ingestr.src.shopify import shopify_source
@@ -1569,3 +1574,80 @@ class GCSSource:
         return readers(
             bucket_url, fs, path_to_file
         ).with_resources(endpoint)
+
+
+class LinkedInAdsSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        source_fields = parse_qs(parsed_uri.query)
+
+        access_token = source_fields.get("access_token")
+        if not access_token:
+            raise ValueError("access_token is required to connect to LinkedIn Ads")
+
+        account_ids = source_fields.get("account_ids")
+
+        if not account_ids:
+            raise ValueError("account_ids is required to connect to LinkedIn Ads")
+        account_ids = account_ids[0].replace(" ", "").split(",")
+
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+        start_date = (
+            ensure_pendulum_datetime(interval_start).date()
+            if interval_start
+            else pendulum.datetime(2018, 1, 1).date()
+        )
+        end_date = (
+            ensure_pendulum_datetime(interval_end).date() if interval_end else None
+        )
+
+        fields = table.split(":")
+        if len(fields) != 3:
+            raise ValueError(
+                "Invalid table format. Expected format: custom:<dimensions>:<metrics>"
+            )
+
+        dimensions = fields[1].replace(" ", "").split(",")
+        dimensions = [item for item in dimensions if item.strip()]
+        if (
+            "campaign" not in dimensions
+            and "creative" not in dimensions
+            and "account" not in dimensions
+        ):
+            raise ValueError(
+                "'campaign', 'creative' or 'account' is required to connect to LinkedIn Ads, please provide at least one of these dimensions."
+            )
+        if "date" not in dimensions and "month" not in dimensions:
+            raise ValueError(
+                "'date' or 'month' is required to connect to LinkedIn Ads, please provide at least one of these dimensions."
+            )
+
+        if "date" in dimensions:
+            time_granularity = TimeGranularity.daily
+            dimensions.remove("date")
+        else:
+            time_granularity = TimeGranularity.monthly
+            dimensions.remove("month")
+
+        dimension = Dimension[dimensions[0]]
+
+        metrics = fields[2].replace(" ", "").split(",")
+        metrics = [item for item in metrics if item.strip()]
+        if "dateRange" not in metrics:
+            metrics.append("dateRange")
+        if "pivotValues" not in metrics:
+            metrics.append("pivotValues")
+
+        return linked_in_ads_source(
+            start_date=start_date,
+            end_date=end_date,
+            access_token=access_token[0],
+            account_ids=account_ids,
+            dimension=dimension,
+            metrics=metrics,
+            time_granularity=time_granularity,
+        ).with_resources("custom_reports")

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.12.9"
+__version__ = "0.12.10"

{ingestr-0.12.9.dist-info → ingestr-0.12.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.9
+Version: 0.12.10
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.12.9.dist-info → ingestr-0.12.10.dist-info}/RECORD

@@ -2,12 +2,12 @@ ingestr/main.py,sha256=fRWnyoPzMvvxTa61EIAP_dsKu0B_0yOwoyt0Slq9WQU,24723
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=zcHJIIHAZmcD9sJomd6G1Bc-1KsxnBD2aByOSV_9L3g,8850
 ingestr/src/errors.py,sha256=MrdLY5Gpr3g3qbYjl-U8-m8kxBJQOJo4ZVOsQpQbRR8,447
-ingestr/src/factory.py,sha256=oNF9dovovLG34xLgRZ5fbyA_XSHxEuTW27s1cb35KDM,4622
+ingestr/src/factory.py,sha256=kzbJ10fF1xySzarhDfJ2l4_Hm925cglsvzk3MAIMkaI,4687
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
-ingestr/src/sources.py,sha256=JoO-IQ_eB4Ia1fC1GWs6N74l9A3tXQT-Fj0uNBiSI_Y,55978
+ingestr/src/sources.py,sha256=qUs9s_0kCurUJUmbrBCTiPzyLpEtaO-yCOnOYvsftUY,58965
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
-ingestr/src/version.py,sha256=FSGqM7DffUSCa5R2rqVlNo-yNzBd6cgAXS1_0tElLy0,23
+ingestr/src/version.py,sha256=037G21EIHmneVX5BgQiyUajkoMsqfZoVvjyP3_6MaDs,24
 ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
 ingestr/src/adjust/adjust_helpers.py,sha256=av97NPSn-hQtTbAC0vUSCAWYePmOiG5R-DGdMssm7FQ,3646
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -54,6 +54,9 @@ ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,
 ingestr/src/klaviyo/_init_.py,sha256=ucWHqBe8DQvXVpbmxKFAV5ljpCFb4ps_2QTD0OSiWxY,7905
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
+ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
+ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
+ingestr/src/linkedin_ads/helpers.py,sha256=6jSIp4DF0iUafJWU3Y7DbIJGKRH6hrx4S7zCTDOjNuE,4528
 ingestr/src/mongodb/__init__.py,sha256=aMr1PFIDUMRv--ne61lR17HudsN-fsrzMeyxe9PqK2s,4335
 ingestr/src/mongodb/helpers.py,sha256=y9rYKR8eyIqam_eNsZmwSYevgi8mghh7Zp8qhTHl65s,5652
 ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
@@ -91,8 +94,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.12.9.dist-info/METADATA,sha256=p7RGcw0cnHPU93RLIPWOkMtj36Ax9BnA7bPSKIQ3pfg,8056
-ingestr-0.12.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.12.9.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.12.9.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.12.9.dist-info/RECORD,,
+ingestr-0.12.10.dist-info/METADATA,sha256=_jr6Mv4lUktQkO2MP3q4RBeHe2RnCy1-4WWGXva6Qbo,8057
+ingestr-0.12.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.12.10.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.12.10.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.12.10.dist-info/RECORD,,