ingestr 0.12.9__py3-none-any.whl → 0.12.11__py3-none-any.whl

ingestr/src/google_ads/reports.py

@@ -0,0 +1,380 @@
+from typing import Dict, List
+
+from . import field
+
+
+class Report:
+    resource: str
+    unfilterable: bool
+    dimensions: List[str]
+    metrics: List[str]
+    segments: List[str]
+
+    def __init__(
+        self,
+        resource: str = "",
+        dimensions: List[str] = [],
+        metrics: List[str] = [],
+        segments: List[str] = [],
+        unfilterable: bool = False,
+    ):
+        self.resource = resource
+        self.dimensions = dimensions
+        self.metrics = metrics
+        self.segments = segments
+        self.unfilterable = unfilterable
+
+    def primary_keys(self) -> List[str]:
+        return [field.to_column(k) for k in self.dimensions + self.segments]
+
+    @classmethod
+    def from_spec(cls, spec: str):
+        """
+        Parse a report specification string into a Report object.
+        The expected format is:
+        custom:{resource}:{dimensions}:{metrics}
+
+        Example:
+        custom:ad_group_ad_asset_view:ad_group.id,campaign.id:clicks,conversions
+        """
+        if spec.count(":") != 3:
+            raise ValueError(
+                "Invalid report specification format. Expected daily:{resource}:{dimensions}:{metrics}"
+            )
+
+        _, resource, dimensions, metrics = spec.split(":")
+
+        report = cls()
+        report.segments = ["segments.date"]
+        report.resource = resource
+        if dimensions.strip() != "":
+            report.dimensions = [
+                d for d in map(cls._parse_dimension, dimensions.split(","))
+            ]
+        if metrics.strip() != "":
+            report.metrics = [m for m in map(cls._parse_metric, metrics.split(","))]
+        return report
+
+    @classmethod
+    def _parse_dimension(self, dim: str):
+        dim = dim.strip()
+        if dim.count(".") == 0:
+            raise ValueError("Invalid dimension format. Expected {resource}.{field}")
+        if dim.startswith("segments."):
+            raise ValueError(
+                "Invalid dimension format. Segments are not allowed in dimensions."
+            )
+        return dim
+
+    @classmethod
+    def _parse_metric(self, metric: str):
+        metric = metric.strip()
+        if not metric.startswith("metrics."):
+            metric = f"metrics.{metric.strip()}"
+        return metric
+
+
+BUILTIN_REPORTS: Dict[str, Report] = {
+    "account_report_daily": Report(
+        resource="campaign",
+        dimensions=[
+            "customer.id",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+    ),
+    "campaign_report_daily": Report(
+        resource="campaign",
+        dimensions=[
+            "campaign.id",
+            "customer.id",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+    ),
+    "ad_group_report_daily": Report(
+        resource="ad_group",
+        dimensions=[
+            "ad_group.id",
+            "customer.id",
+            "campaign.id",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+    ),
+    "ad_report_daily": Report(
+        resource="ad_group_ad",
+        dimensions=[
+            "ad_group.id",
+            "ad_group_ad.ad.id",
+            "customer.id",
+            "campaign.id",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+    ),
+    "audience_report_daily": Report(
+        resource="ad_group_audience_view",
+        dimensions=[
+            "ad_group.id",
+            "customer.id",
+            "campaign.id",
+            "ad_group_criterion.criterion_id",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+    ),
+    "keyword_report_daily": Report(
+        resource="keyword_view",
+        dimensions=[
+            "ad_group.id",
+            "customer.id",
+            "campaign.id",
+            "ad_group_criterion.criterion_id",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+    ),
+    "click_report_daily": Report(
+        resource="click_view",
+        dimensions=[
+            "click_view.gclid",
+            "customer.id",
+            "ad_group.id",
+            "campaign.id",
+            "segments.date",
+        ],
+        metrics=[
+            "metrics.clicks",
+        ],
+    ),
+    "landing_page_report_daily": Report(
+        resource="landing_page_view",
+        dimensions=[
+            "landing_page_view.unexpanded_final_url",
+            "landing_page_view.resource_name",
+            "customer.id",
+            "ad_group.id",
+            "campaign.id",
+            "segments.date",
+        ],
+        metrics=[
+            "metrics.average_cpc",
+            "metrics.clicks",
+            "metrics.cost_micros",
+            "metrics.ctr",
+            "metrics.impressions",
+            "metrics.mobile_friendly_clicks_percentage",
+            "metrics.speed_score",
+            "metrics.valid_accelerated_mobile_pages_clicks_percentage",
+        ],
+    ),
+    "search_keyword_report_daily": Report(
+        resource="keyword_view",
+        dimensions=[
+            "customer.id",
+            "ad_group.id",
+            "campaign.id",
+            "keyword_view.resource_name",
+            "ad_group_criterion.criterion_id",
+            "segments.date",
+        ],
+        metrics=[
+            "metrics.absolute_top_impression_percentage",
+            "metrics.average_cpc",
+            "metrics.average_cpm",
+            "metrics.clicks",
+            "metrics.conversions_from_interactions_rate",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.ctr",
+            "metrics.impressions",
+            "metrics.top_impression_percentage",
+            "metrics.view_through_conversions",
+        ],
+    ),
+    "search_term_report_daily": Report(
+        resource="search_term_view",
+        dimensions=[
+            "customer.id",
+            "ad_group.id",
+            "campaign.id",
+            "search_term_view.resource_name",
+            "search_term_view.search_term",
+            "search_term_view.status",
+            "segments.date",
+        ],
+        segments=[
+            "segments.search_term_match_type",
+        ],
+        metrics=[
+            "metrics.absolute_top_impression_percentage",
+            "metrics.average_cpc",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_from_interactions_rate",
+            "metrics.conversions_from_interactions_value_per_interaction",
+            "metrics.cost_micros",
+            "metrics.ctr",
+            "metrics.impressions",
+            "metrics.top_impression_percentage",
+            "metrics.view_through_conversions",
+        ],
+    ),
+    "lead_form_submission_data_report_daily": Report(
+        resource="lead_form_submission_data",
+        dimensions=[
+            "lead_form_submission_data.gclid",
+            "lead_form_submission_data.submission_date_time",
+            "lead_form_submission_data.lead_form_submission_fields",
+            "lead_form_submission_data.custom_lead_form_submission_fields",
+            "lead_form_submission_data.resource_name",
+            "customer.id",
+            "ad_group_ad.ad.id",
+            "ad_group.id",
+            "campaign.id",
+        ],
+        unfilterable=True,
+    ),
+    "local_services_lead_report_daily": Report(
+        resource="local_services_lead",
+        dimensions=[
+            "customer.id",
+            "local_services_lead.creation_date_time",
+            "local_services_lead.contact_details",
+            "local_services_lead.credit_details.credit_state",
+            "local_services_lead.credit_details.credit_state_last_update_date_time",
+            "local_services_lead.lead_charged",
+            "local_services_lead.lead_status",
+            "local_services_lead.lead_type",
+            "local_services_lead.locale",
+            "local_services_lead.note.description",
+            "local_services_lead.note.edit_date_time",
+            "local_services_lead.service_id",
+        ],
+        unfilterable=True,
+    ),
+    "local_services_lead_conversations_report_daily": Report(
+        resource="local_services_lead_conversation",
+        dimensions=[
+            "customer.id",
+            "local_services_lead_conversation.id",
+            "local_services_lead_conversation.event_date_time",
+            "local_services_lead_conversation.conversation_channel",
+            "local_services_lead_conversation.message_details.attachment_urls",
+            "local_services_lead_conversation.message_details.text",
+            "local_services_lead_conversation.participant_type",
+            "local_services_lead_conversation.phone_call_details.call_duration_millis",
+            "local_services_lead_conversation.phone_call_details.call_recording_url",
+        ],
+        unfilterable=True,
+    ),
+}

ingestr/src/linkedin_ads/__init__.py

@@ -0,0 +1,63 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from pendulum import Date
+
+from .dimension_time_enum import Dimension, TimeGranularity
+from .helpers import LinkedInAdsAPI, find_intervals
+
+
+@dlt.source(max_table_nesting=0)
+def linked_in_ads_source(
+    start_date: Date,
+    end_date: Date | None,
+    access_token: str,
+    account_ids: list[str],
+    dimension: Dimension,
+    metrics: list[str],
+    time_granularity: TimeGranularity,
+) -> DltResource:
+    linkedin_api = LinkedInAdsAPI(
+        access_token=access_token,
+        account_ids=account_ids,
+        dimension=dimension,
+        metrics=metrics,
+        time_granularity=time_granularity,
+    )
+
+    if time_granularity == TimeGranularity.daily:
+        primary_key = [dimension.value, "date"]
+        incremental_loading_param = "date"
+    else:
+        primary_key = [dimension.value, "start_date", "end_date"]
+        incremental_loading_param = "start_date"
+
+    @dlt.resource(write_disposition="merge", primary_key=primary_key)
+    def custom_reports(
+        dateTime=(
+            dlt.sources.incremental(
+                incremental_loading_param,
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterable[TDataItem]:
+        if dateTime.end_value is None:
+            end_date = pendulum.now().date()
+        else:
+            end_date = dateTime.end_value
+
+        list_of_interval = find_intervals(
+            start_date=dateTime.last_value,
+            end_date=end_date,
+            time_granularity=time_granularity,
+        )
+        for start, end in list_of_interval:
+            yield linkedin_api.fetch_pages(start, end)
+
+    return custom_reports

ingestr/src/linkedin_ads/dimension_time_enum.py

@@ -0,0 +1,12 @@
+from enum import Enum
+
+
+class Dimension(Enum):
+    campaign = "campaign"
+    creative = "creative"
+    account = "account"
+
+
+class TimeGranularity(Enum):
+    daily = "DAILY"
+    monthly = "MONTHLY"

ingestr/src/linkedin_ads/helpers.py

@@ -0,0 +1,148 @@
+from urllib.parse import quote
+
+import pendulum
+import requests
+from dlt.sources.helpers.requests import Client
+from pendulum import Date
+
+from .dimension_time_enum import Dimension, TimeGranularity
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 429
+
+
+def create_client() -> requests.Session:
+    return Client(
+        request_timeout=10.0,
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+    ).session
+
+
+def flat_structure(items, pivot: Dimension, time_granularity: TimeGranularity):
+    for item in items:
+        if "pivotValues" in item:
+            if len(item["pivotValues"]) > 1:
+                item[pivot.value.lower()] = item["pivotValues"]
+            else:
+                item[pivot.value.lower()] = item["pivotValues"][0]
+        if "dateRange" in item:
+            start_date = item["dateRange"]["start"]
+            start_dt = pendulum.date(
+                year=start_date["year"],
+                month=start_date["month"],
+                day=start_date["day"],
+            )
+            if time_granularity == TimeGranularity.daily:
+                item["date"] = start_dt
+            else:
+                end_date = item["dateRange"]["end"]
+                end_dt = pendulum.date(
+                    year=end_date["year"],
+                    month=end_date["month"],
+                    day=end_date["day"],
+                )
+                item["start_date"] = start_dt
+                item["end_date"] = end_dt
+
+        del item["dateRange"]
+        del item["pivotValues"]
+
+    return items
+
+
+def find_intervals(start_date: Date, end_date: Date, time_granularity: TimeGranularity):
+    intervals = []
+
+    if start_date > end_date:
+        raise ValueError("Start date must be less than end date")
+
+    while start_date <= end_date:
+        if time_granularity == TimeGranularity.daily:
+            next_date = min(start_date.add(months=6), end_date)
+        else:
+            next_date = min(start_date.add(years=2), end_date)
+
+        intervals.append((start_date, next_date))
+
+        start_date = next_date.add(days=1)
+
+    return intervals
+
+
+def construct_url(
+    start: Date,
+    end: Date,
+    account_ids: list[str],
+    metrics: list[str],
+    dimension: Dimension,
+    time_granularity: TimeGranularity,
+):
+    date_range = f"(start:(year:{start.year},month:{start.month},day:{start.day})"
+    date_range += f",end:(year:{end.year},month:{end.month},day:{end.day}))"
+    accounts = ",".join(
+        [quote(f"urn:li:sponsoredAccount:{account_id}") for account_id in account_ids]
+    )
+    encoded_accounts = f"List({accounts})"
+    dimension_str = dimension.value.upper()
+    time_granularity_str = time_granularity.value
+    metrics_str = ",".join([metric for metric in metrics])
+
+    url = (
+        f"https://api.linkedin.com/rest/adAnalytics?"
+        f"q=analytics&timeGranularity={time_granularity_str}&"
+        f"dateRange={date_range}&accounts={encoded_accounts}&"
+        f"pivot={dimension_str}&fields={metrics_str}"
+    )
+
+    return url
+
+
+class LinkedInAdsAPI:
+    def __init__(
+        self,
+        access_token,
+        time_granularity,
+        account_ids,
+        dimension,
+        metrics,
+    ):
+        self.time_granularity: TimeGranularity = time_granularity
+        self.account_ids: list[str] = account_ids
+        self.dimension: Dimension = dimension
+        self.metrics: list[str] = metrics
+        self.headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Linkedin-Version": "202411",
+            "X-Restli-Protocol-Version": "2.0.0",
+        }
+
+    def fetch_pages(self, start: Date, end: Date):
+        client = create_client()
+        url = construct_url(
+            start=start,
+            end=end,
+            account_ids=self.account_ids,
+            metrics=self.metrics,
+            dimension=self.dimension,
+            time_granularity=self.time_granularity,
+        )
+        response = client.get(url=url, headers=self.headers)
+
+        if response.status_code != 200:
+            error_data = response.json()
+            raise ValueError(f"LinkedIn API Error: {error_data.get('message')}")
+
+        result = response.json()
+        items = result.get("elements", [])
+        yield flat_structure(
+            items=items,
+            pivot=self.dimension,
+            time_granularity=self.time_granularity,
+        )