omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/google_ads/reports.py

@@ -0,0 +1,411 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List
+
+from . import field
+
+
+class Report:
+    resource: str
+    unfilterable: bool
+    dimensions: List[str]
+    metrics: List[str]
+    segments: List[str]
+
+    def __init__(
+        self,
+        resource: str = "",
+        dimensions: List[str] = [],
+        metrics: List[str] = [],
+        segments: List[str] = [],
+        unfilterable: bool = False,
+    ):
+        self.resource = resource
+        self.dimensions = dimensions
+        self.metrics = metrics
+        self.segments = segments
+        self.unfilterable = unfilterable
+
+    def primary_keys(self) -> List[str]:
+        dims = self.dimensions + self.segments
+        filtered = []
+        for d in dims:
+            filtered.append(field.to_column(d))
+        return filtered
+
+    @classmethod
+    def from_spec(cls, spec: str):
+        """
+        Parse a report specification string into a Report object.
+        The expected format is:
+        daily:{resource}:{dimensions}:{metrics}:{customer_ids}
+
+        Example:
+        daily:ad_group_ad_asset_view:ad_group.id,campaign.id:clicks,conversions:1234567890,9876543210
+
+        customer_ids is optional - if not provided, falls back to URI hostname.
+        """
+        colon_count = spec.count(":")
+        if colon_count not in (3, 4):
+            raise ValueError(
+                "Invalid report specification format. Expected daily:{resource}:{dimensions}:{metrics} or daily:{resource}:{dimensions}:{metrics}:{customer_ids}"
+            )
+
+        parts = spec.split(":")
+        _, resource, dimensions, metrics = parts[:4]
+        customer_ids_str = parts[4] if len(parts) > 4 else None
+
+        if dimensions.strip() == "":
+            raise ValueError("Dimensions are required")
+        if metrics.strip() == "":
+            raise ValueError("Metrics are required")
+
+        report = cls()
+        report.segments = ["segments.date"]
+        report.resource = resource
+        report.dimensions = [
+            d for d in map(cls._parse_dimension, dimensions.split(","))
+        ]
+        report.metrics = [m for m in map(cls._parse_metric, metrics.split(","))]
+
+        customer_ids = None
+        if customer_ids_str and customer_ids_str.strip() != "":
+            customer_ids = [cid.strip() for cid in customer_ids_str.split(",")]
+
+        return report, customer_ids
+
+    @classmethod
+    def _parse_dimension(self, dim: str):
+        dim = dim.strip()
+        if dim.count(".") == 0:
+            raise ValueError("Invalid dimension format. Expected {resource}.{field}")
+        if dim.startswith("segments."):
+            raise ValueError(
+                "Invalid dimension format. Segments are not allowed in dimensions."
+            )
+        return dim
+
+    @classmethod
+    def _parse_metric(self, metric: str):
+        metric = metric.strip()
+        if not metric.startswith("metrics."):
+            metric = f"metrics.{metric.strip()}"
+        return metric
+
+
+BUILTIN_REPORTS: Dict[str, Report] = {
+    "account_report_daily": Report(
+        resource="campaign",
+        dimensions=[
+            "customer.id",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+    ),
+    "campaign_report_daily": Report(
+        resource="campaign",
+        dimensions=[
+            "campaign.id",
+            "customer.id",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+    ),
+    "ad_group_report_daily": Report(
+        resource="ad_group",
+        dimensions=[
+            "ad_group.id",
+            "customer.id",
+            "campaign.id",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+    ),
+    "ad_report_daily": Report(
+        resource="ad_group_ad",
+        dimensions=[
+            "ad_group.id",
+            "ad_group_ad.ad.id",
+            "customer.id",
+            "campaign.id",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+    ),
+    "audience_report_daily": Report(
+        resource="ad_group_audience_view",
+        dimensions=[
+            "ad_group.id",
+            "customer.id",
+            "campaign.id",
+            "ad_group_criterion.criterion_id",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+    ),
+    "keyword_report_daily": Report(
+        resource="keyword_view",
+        dimensions=[
+            "ad_group.id",
+            "customer.id",
+            "campaign.id",
+            "ad_group_criterion.criterion_id",
+        ],
+        segments=[
+            "segments.date",
+            "segments.ad_network_type",
+            "segments.device",
+        ],
+        metrics=[
+            "metrics.active_view_impressions",
+            "metrics.active_view_measurability",
+            "metrics.active_view_measurable_cost_micros",
+            "metrics.active_view_measurable_impressions",
+            "metrics.active_view_viewability",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.impressions",
+            "metrics.interactions",
+            "metrics.interaction_event_types",
+            "metrics.view_through_conversions",
+        ],
+    ),
+    "click_report_daily": Report(
+        resource="click_view",
+        dimensions=[
+            "click_view.gclid",
+            "customer.id",
+            "ad_group.id",
+            "campaign.id",
+            "segments.date",
+        ],
+        metrics=[
+            "metrics.clicks",
+        ],
+    ),
+    "landing_page_report_daily": Report(
+        resource="landing_page_view",
+        dimensions=[
+            "landing_page_view.unexpanded_final_url",
+            "landing_page_view.resource_name",
+            "customer.id",
+            "ad_group.id",
+            "campaign.id",
+            "segments.date",
+        ],
+        metrics=[
+            "metrics.average_cpc",
+            "metrics.clicks",
+            "metrics.cost_micros",
+            "metrics.ctr",
+            "metrics.impressions",
+            "metrics.mobile_friendly_clicks_percentage",
+            "metrics.speed_score",
+            "metrics.valid_accelerated_mobile_pages_clicks_percentage",
+        ],
+    ),
+    "search_keyword_report_daily": Report(
+        resource="keyword_view",
+        dimensions=[
+            "customer.id",
+            "ad_group.id",
+            "campaign.id",
+            "keyword_view.resource_name",
+            "ad_group_criterion.criterion_id",
+            "segments.date",
+        ],
+        metrics=[
+            "metrics.absolute_top_impression_percentage",
+            "metrics.average_cpc",
+            "metrics.average_cpm",
+            "metrics.clicks",
+            "metrics.conversions_from_interactions_rate",
+            "metrics.conversions_value",
+            "metrics.cost_micros",
+            "metrics.ctr",
+            "metrics.impressions",
+            "metrics.top_impression_percentage",
+            "metrics.view_through_conversions",
+        ],
+    ),
+    "search_term_report_daily": Report(
+        resource="search_term_view",
+        dimensions=[
+            "customer.id",
+            "ad_group.id",
+            "campaign.id",
+            "search_term_view.resource_name",
+            "search_term_view.search_term",
+            "search_term_view.status",
+            "segments.date",
+        ],
+        segments=[
+            "segments.search_term_match_type",
+        ],
+        metrics=[
+            "metrics.absolute_top_impression_percentage",
+            "metrics.average_cpc",
+            "metrics.clicks",
+            "metrics.conversions",
+            "metrics.conversions_from_interactions_rate",
+            "metrics.conversions_from_interactions_value_per_interaction",
+            "metrics.cost_micros",
+            "metrics.ctr",
+            "metrics.impressions",
+            "metrics.top_impression_percentage",
+            "metrics.view_through_conversions",
+        ],
+    ),
+    "lead_form_submission_data_report_daily": Report(
+        resource="lead_form_submission_data",
+        dimensions=[
+            "lead_form_submission_data.gclid",
+            "lead_form_submission_data.submission_date_time",
+            "lead_form_submission_data.lead_form_submission_fields",
+            "lead_form_submission_data.custom_lead_form_submission_fields",
+            "lead_form_submission_data.resource_name",
+            "customer.id",
+            "ad_group_ad.ad.id",
+            "ad_group.id",
+            "campaign.id",
+        ],
+        unfilterable=True,
+    ),
+    "local_services_lead_report_daily": Report(
+        resource="local_services_lead",
+        dimensions=[
+            "customer.id",
+            "local_services_lead.creation_date_time",
+            "local_services_lead.contact_details",
+            "local_services_lead.credit_details.credit_state",
+            "local_services_lead.credit_details.credit_state_last_update_date_time",
+            "local_services_lead.lead_charged",
+            "local_services_lead.lead_status",
+            "local_services_lead.lead_type",
+            "local_services_lead.locale",
+            "local_services_lead.note.description",
+            "local_services_lead.note.edit_date_time",
+            "local_services_lead.service_id",
+        ],
+        unfilterable=True,
+    ),
+    "local_services_lead_conversations_report_daily": Report(
+        resource="local_services_lead_conversation",
+        dimensions=[
+            "customer.id",
+            "local_services_lead_conversation.id",
+            "local_services_lead_conversation.event_date_time",
+            "local_services_lead_conversation.conversation_channel",
+            "local_services_lead_conversation.message_details.attachment_urls",
+            "local_services_lead_conversation.message_details.text",
+            "local_services_lead_conversation.participant_type",
+            "local_services_lead_conversation.phone_call_details.call_duration_millis",
+            "local_services_lead_conversation.phone_call_details.call_recording_url",
+        ],
+        unfilterable=True,
+    ),
+}

omniload/src/google_ads/test_google_ads.py

@@ -0,0 +1,184 @@
+from omniload.src.google_ads import extract_fields
+
+FIELD_PATHS = [
+    "customer.id",
+    "campaign.id",
+    "campaign.name",
+    "ad_group.id",
+    "ad_group.name",
+    "ad_group_ad.resource_name",
+    "ad_group_ad.status",
+    "ad_group_ad.ad.id",
+    "ad_group_ad.ad.type",
+    "ad_group_ad.ad.name",
+    "ad_group_ad.ad.final_urls",
+    "ad_group_ad.ad.responsive_search_ad.path1",
+    "ad_group_ad.ad.responsive_search_ad.path2",
+    "ad_group_ad.ad.responsive_display_ad.long_headline",
+    "ad_group_ad.ad.responsive_display_ad.call_to_action_text",
+    "ad_group_ad.ad.responsive_display_ad.format_setting",
+    "ad_group_ad.ad.responsive_display_ad.headlines",
+    "ad_group_ad.ad.responsive_display_ad.descriptions",
+]
+
+EXPECTED_KEYS = {path.replace(".", "_") for path in FIELD_PATHS}
+
+
+def test_extract_fields():
+    display_ad_data = {
+        "customer": {
+            "resource_name": "customers/1234567890",
+            "id": "1234567890",
+        },
+        "campaign": {
+            "resource_name": "customers/1234567890/campaigns/111",
+            "name": "Summer Display Campaign",
+            "id": "111",
+        },
+        "ad_group": {
+            "resource_name": "customers/1234567890/adGroups/222",
+            "id": "222",
+            "name": "Display Ad Group",
+        },
+        "ad_group_ad": {
+            "resource_name": "customers/1234567890/adGroupAds/222~333",
+            "status": "ENABLED",
+            "ad": {
+                "type": "RESPONSIVE_DISPLAY_AD",
+                "responsive_display_ad": {
+                    "headlines": [{"text": "Buy Now"}],
+                    "long_headline": {"text": "Great deals on summer products"},
+                    "descriptions": [
+                        {"text": "Free shipping on all orders"},
+                        {"text": "Limited time offer"},
+                    ],
+                    "format_setting": "ALL_FORMATS",
+                },
+                "resource_name": "customers/1234567890/ads/333",
+                "id": "333",
+                "final_urls": ["https://example.com/summer"],
+            },
+        },
+    }
+
+    call_ad_data = {
+        "customer": {
+            "resource_name": "customers/1234567890",
+            "id": "1234567890",
+        },
+        "campaign": {
+            "resource_name": "customers/1234567890/campaigns/444",
+            "name": "Call Campaign",
+            "id": "444",
+        },
+        "ad_group": {
+            "resource_name": "customers/1234567890/adGroups/555",
+            "id": "555",
+            "name": "Call Ad Group",
+        },
+        "ad_group_ad": {
+            "resource_name": "customers/1234567890/adGroupAds/555~666",
+            "status": "PAUSED",
+            "ad": {
+                "type": "CALL_AD",
+                "resource_name": "customers/1234567890/ads/666",
+                "id": "666",
+                "final_urls": ["https://example.com/contact"],
+            },
+        },
+    }
+
+    search_ad_data = {
+        "customer": {
+            "resource_name": "customers/1234567890",
+            "id": "1234567890",
+        },
+        "campaign": {
+            "resource_name": "customers/1234567890/campaigns/777",
+            "name": "Search Campaign",
+            "id": "777",
+        },
+        "ad_group": {
+            "resource_name": "customers/1234567890/adGroups/888",
+            "id": "888",
+            "name": "Search Ad Group",
+        },
+        "ad_group_ad": {
+            "resource_name": "customers/1234567890/adGroupAds/888~999",
+            "status": "PAUSED",
+            "ad": {
+                "type": "RESPONSIVE_SEARCH_AD",
+                "responsive_search_ad": {
+                    "path1": "deals",
+                    "path2": "today",
+                },
+                "resource_name": "customers/1234567890/ads/999",
+                "id": "999",
+                "final_urls": ["https://example.com/search"],
+            },
+        },
+    }
+
+    for row_data in [display_ad_data, call_ad_data, search_ad_data]:
+        result = extract_fields(row_data, FIELD_PATHS)
+        assert set(result.keys()) == EXPECTED_KEYS
+
+    # display ad
+    display = extract_fields(display_ad_data, FIELD_PATHS)
+    assert display["customer_id"] == "1234567890"
+    assert display["campaign_id"] == "111"
+    assert display["campaign_name"] == "Summer Display Campaign"
+    assert display["ad_group_id"] == "222"
+    assert display["ad_group_name"] == "Display Ad Group"
+    assert (
+        display["ad_group_ad_resource_name"]
+        == "customers/1234567890/adGroupAds/222~333"
+    )
+    assert display["ad_group_ad_status"] == "ENABLED"
+    assert display["ad_group_ad_ad_id"] == "333"
+    assert display["ad_group_ad_ad_type"] == "RESPONSIVE_DISPLAY_AD"
+    assert display["ad_group_ad_ad_final_urls"] == ["https://example.com/summer"]
+    assert display["ad_group_ad_ad_responsive_display_ad_headlines"] == [
+        {"text": "Buy Now"}
+    ]
+    assert display["ad_group_ad_ad_responsive_display_ad_long_headline"] == {
+        "text": "Great deals on summer products"
+    }
+    assert display["ad_group_ad_ad_responsive_display_ad_descriptions"] == [
+        {"text": "Free shipping on all orders"},
+        {"text": "Limited time offer"},
+    ]
+    assert (
+        display["ad_group_ad_ad_responsive_display_ad_format_setting"] == "ALL_FORMATS"
+    )
+    assert display["ad_group_ad_ad_responsive_search_ad_path1"] is None
+    assert display["ad_group_ad_ad_responsive_search_ad_path2"] is None
+    assert display["ad_group_ad_ad_name"] is None
+
+    # call ad
+    call = extract_fields(call_ad_data, FIELD_PATHS)
+    assert call["customer_id"] == "1234567890"
+    assert call["campaign_name"] == "Call Campaign"
+    assert call["ad_group_ad_status"] == "PAUSED"
+    assert call["ad_group_ad_ad_type"] == "CALL_AD"
+    assert call["ad_group_ad_ad_id"] == "666"
+    assert call["ad_group_ad_ad_final_urls"] == ["https://example.com/contact"]
+    assert call["ad_group_ad_ad_responsive_display_ad_headlines"] is None
+    assert call["ad_group_ad_ad_responsive_display_ad_descriptions"] is None
+    assert call["ad_group_ad_ad_responsive_display_ad_long_headline"] is None
+    assert call["ad_group_ad_ad_responsive_search_ad_path1"] is None
+    assert call["ad_group_ad_ad_responsive_search_ad_path2"] is None
+    assert call["ad_group_ad_ad_name"] is None
+
+    # search ad
+    search = extract_fields(search_ad_data, FIELD_PATHS)
+    assert search["customer_id"] == "1234567890"
+    assert search["campaign_name"] == "Search Campaign"
+    assert search["ad_group_ad_ad_type"] == "RESPONSIVE_SEARCH_AD"
+    assert search["ad_group_ad_ad_responsive_search_ad_path1"] == "deals"
+    assert search["ad_group_ad_ad_responsive_search_ad_path2"] == "today"
+    assert search["ad_group_ad_ad_final_urls"] == ["https://example.com/search"]
+    assert search["ad_group_ad_ad_responsive_display_ad_headlines"] is None
+    assert search["ad_group_ad_ad_responsive_display_ad_descriptions"] is None
+    assert search["ad_group_ad_ad_responsive_display_ad_long_headline"] is None
+    assert search["ad_group_ad_ad_name"] is None