omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/github/queries.py

@@ -0,0 +1,129 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+RATE_LIMIT = """
+  rateLimit {
+    limit
+    cost
+    remaining
+    resetAt
+  }
+"""
+
+ISSUES_QUERY = """
+query($owner: String!, $name: String!, $issues_per_page: Int!, $first_reactions: Int!, $first_comments: Int!, $page_after: String) {
+  repository(owner: $owner, name: $name) {
+    %s(first: $issues_per_page, orderBy: {field: CREATED_AT, direction: DESC}, after: $page_after) {
+      totalCount
+      pageInfo {
+        endCursor
+        startCursor
+      }
+      nodes {
+        # id
+        number
+        url
+        title
+        body
+        author {login avatarUrl url}
+        authorAssociation
+        closed
+        closedAt
+        createdAt
+        state
+        updatedAt
+        reactions(first: $first_reactions) {
+          totalCount
+          nodes {
+            # id
+            user {login avatarUrl url}
+            content
+            createdAt
+          }
+        }
+        comments(first: $first_comments) {
+          totalCount
+          nodes {
+            id
+            url
+            body
+            author {avatarUrl login url}
+            authorAssociation
+            createdAt
+            reactionGroups {content createdAt}
+            # reactions(first: 0) {
+            #   totalCount
+            #   nodes {
+            #     # id
+            #     user {login avatarUrl url}
+            #     content
+            #     createdAt
+            #   }
+            # }
+          }
+        }
+      }
+    }
+  }
+  rateLimit {
+    limit
+    cost
+    remaining
+    resetAt
+  }
+}
+"""
+
+COMMENT_REACTIONS_QUERY = """
+node_%s: node(id:"%s") {
+     ... on IssueComment {
+      id
+      reactions(first: 100) {
+        totalCount
+        nodes {
+            user {login avatarUrl url}
+            content
+            createdAt
+          }
+      }
+    }
+  }
+"""
+
+STARGAZERS_QUERY = """
+query($owner: String!, $name: String!, $items_per_page: Int!, $page_after: String) {
+  repository(owner: $owner, name: $name) {
+    stargazers(first: $items_per_page, orderBy: {field: STARRED_AT, direction: DESC}, after: $page_after) {
+      pageInfo {
+        endCursor
+        startCursor
+      }
+      edges {
+        starredAt
+        node {
+          login
+          avatarUrl
+          url
+        }
+      }
+    }
+  }
+  rateLimit {
+    limit
+    cost
+    remaining
+    resetAt
+  }
+}
+"""

omniload/src/github/settings.py

@@ -0,0 +1,24 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Github source settings and constants."""
+
+START_DATE = "1970-01-01T00:00:00Z"
+
+# rest queries
+REST_API_BASE_URL = "https://api.github.com"
+REPO_EVENTS_PATH = "/repos/%s/%s/events"
+
+# graphql queries
+GRAPHQL_API_BASE_URL = "https://api.github.com/graphql"

omniload/src/google_ads/__init__.py

@@ -0,0 +1,198 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from datetime import date, datetime
+from typing import Any, Iterator, Optional
+
+import dlt
+import proto  # type: ignore
+from dlt.common.exceptions import MissingDependencyException
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from flatten_json import flatten  # type: ignore
+from googleapiclient.discovery import Resource  # type: ignore
+
+from . import field
+from .metrics import dlt_metrics_schema
+from .predicates import date_predicate
+from .reports import BUILTIN_REPORTS, Report
+
+try:
+    from google.ads.googleads.client import GoogleAdsClient  # type: ignore
+except ImportError:
+    raise MissingDependencyException("Requests-OAuthlib", ["google-ads"])
+
+
+@dlt.source
+def google_ads(
+    client: GoogleAdsClient,
+    customer_ids: list[str],
+    report_spec: Optional[str] = None,
+    gaql_query: Optional[str] = None,
+    start_date: Optional[datetime] = None,
+    end_date: Optional[datetime] = None,
+) -> Iterator[DltResource]:
+    date_range = dlt.sources.incremental(
+        "segments_date",
+        initial_value=start_date.date(),  # type: ignore
+        end_value=end_date.date() if end_date is not None else None,  # type: ignore
+        range_start="closed",
+        range_end="closed",
+    )
+    if report_spec is not None:
+        custom_report, _ = Report.from_spec(report_spec)
+        yield dlt.resource(
+            daily_report,
+            name="daily_report",
+            write_disposition="merge",
+            primary_key=custom_report.primary_keys() + ["customer_id"],
+            columns=dlt_metrics_schema(custom_report.metrics),
+        )(client, customer_ids, custom_report, date_range)
+
+    if gaql_query is not None:
+        yield dlt.resource(
+            run_gaql_query,
+            name="gaql_query",
+            write_disposition="append",
+            max_table_nesting=0,
+        )(client, customer_ids, gaql_query, start_date, end_date)
+
+    for report_name, report in BUILTIN_REPORTS.items():
+        yield dlt.resource(
+            daily_report,
+            name=report_name,
+            write_disposition="merge",
+            primary_key=report.primary_keys() + ["customer_id"],
+            columns=dlt_metrics_schema(report.metrics),
+        )(client, customer_ids, report, date_range)
+
+
+def daily_report(
+    client: Resource,
+    customer_ids: list[str],
+    report: Report,
+    date: dlt.sources.incremental[date],
+) -> Iterator[TDataItem]:
+    ga_service = client.get_service("GoogleAdsService")
+    fields = report.dimensions + report.metrics + report.segments
+    criteria = date_predicate("segments.date", date.last_value, date.end_value)  # type:ignore
+    query = f"""
+        SELECT
+            {", ".join(fields)}
+        FROM
+            {report.resource}
+        WHERE
+            {criteria}
+    """
+    if report.unfilterable is True:
+        i = query.index("WHERE", 0)
+        query = query[:i]
+
+    allowed_keys = set([field.to_column(k) for k in fields])
+    for customer_id in customer_ids:
+        stream = ga_service.search_stream(customer_id=customer_id, query=query)
+        for batch in stream:
+            for row in batch.results:
+                data = flatten(merge_lists(to_dict(row)))
+                if "segments_date" in data:
+                    data["segments_date"] = datetime.strptime(
+                        data["segments_date"], "%Y-%m-%d"
+                    ).date()
+                row_data = {k: v for k, v in data.items() if k in allowed_keys}
+                for pk in report.primary_keys():
+                    if pk not in row_data or row_data[pk] is None or row_data[pk] == "":
+                        row_data[pk] = "-"
+                row_data["customer_id"] = customer_id
+                yield row_data
+
+
+def to_dict(item: Any) -> TDataItem:
+    """
+    Processes a batch result (page of results per dimension) accordingly
+    :param batch:
+    :return:
+    """
+    return json.loads(
+        proto.Message.to_json(
+            item,
+            preserving_proto_field_name=True,
+            use_integers_for_enums=False,
+            including_default_value_fields=False,
+        )
+    )
+
+
+def merge_lists(item: dict) -> dict:
+    replacements = {}
+    for k, v in item.get("metrics", {}).items():
+        if isinstance(v, list):
+            replacements[k] = ",".join(v)
+    if len(replacements) == 0:
+        return item
+    item["metrics"].update(replacements)
+    return item
+
+
+def extract_fields(data: dict, field_paths: list[str]) -> dict:
+    result = {}
+    for path in field_paths:
+        parts = path.split(".")
+        value: Any = data
+        for part in parts:
+            if isinstance(value, dict):
+                value = value.get(part) if part in value else value.get(part + "_")
+            else:
+                value = None
+                break
+        column_name = path.replace(".", "_")
+        result[column_name] = value
+    return result
+
+
+def run_gaql_query(
+    client: GoogleAdsClient,
+    customer_ids: list[str],
+    query: str,
+    start_date: Optional[datetime] = None,
+    end_date: Optional[datetime] = None,
+) -> Iterator[TDataItem]:
+    """
+    Execute a raw Google Ads Query Language (GAQL) query.
+    Supports :interval_start and :interval_end placeholders for date filtering.
+    """
+    ga_service = client.get_service("GoogleAdsService")
+
+    if ":interval_start" in query:
+        start_str = start_date.strftime("%Y-%m-%d") if start_date else "1970-01-01"
+        query = query.replace(":interval_start", f"'{start_str}'")
+
+    if ":interval_end" in query:
+        end_str = (
+            end_date.strftime("%Y-%m-%d")
+            if end_date
+            else date.today().strftime("%Y-%m-%d")
+        )
+        query = query.replace(":interval_end", f"'{end_str}'")
+
+    field_paths = None
+    for customer_id in customer_ids:
+        stream = ga_service.search_stream(customer_id=customer_id, query=query)
+        for batch in stream:
+            if field_paths is None:
+                field_paths = list(batch.field_mask.paths)
+            for row in batch.results:
+                data = extract_fields(to_dict(row), field_paths)
+                data["customer_id"] = customer_id
+                yield data

omniload/src/google_ads/field.py

@@ -0,0 +1,17 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def to_column(field: str) -> str:
+    return field.replace(".", "_")

omniload/src/google_ads/metrics.py

@@ -0,0 +1,254 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from . import field
+
+METRICS_SCHEMA = {
+    "metrics.absolute_top_impression_percentage": "DOUBLE",
+    "metrics.active_view_cpm": "DOUBLE",
+    "metrics.active_view_ctr": "DOUBLE",
+    "metrics.active_view_impressions": "INT64",
+    "metrics.active_view_measurability": "DOUBLE",
+    "metrics.active_view_measurable_cost_micros": "INT64",
+    "metrics.active_view_measurable_impressions": "INT64",
+    "metrics.active_view_viewability": "DOUBLE",
+    "metrics.all_conversions": "DOUBLE",
+    "metrics.all_conversions_by_conversion_date": "DOUBLE",
+    "metrics.all_conversions_from_click_to_call": "DOUBLE",
+    "metrics.all_conversions_from_directions": "DOUBLE",
+    "metrics.all_conversions_from_interactions_rate": "DOUBLE",
+    "metrics.all_conversions_from_interactions_value_per_interaction": "DOUBLE",
+    "metrics.all_conversions_from_location_asset_click_to_call": "DOUBLE",
+    "metrics.all_conversions_from_location_asset_directions": "DOUBLE",
+    "metrics.all_conversions_from_location_asset_menu": "DOUBLE",
+    "metrics.all_conversions_from_location_asset_order": "DOUBLE",
+    "metrics.all_conversions_from_location_asset_other_engagement": "DOUBLE",
+    "metrics.all_conversions_from_location_asset_store_visits": "DOUBLE",
+    "metrics.all_conversions_from_location_asset_website": "DOUBLE",
+    "metrics.all_conversions_from_menu": "DOUBLE",
+    "metrics.all_conversions_from_order": "DOUBLE",
+    "metrics.all_conversions_from_other_engagement": "DOUBLE",
+    "metrics.all_conversions_from_store_visit": "DOUBLE",
+    "metrics.all_conversions_from_store_website": "DOUBLE",
+    "metrics.all_conversions_value": "DOUBLE",
+    "metrics.all_conversions_value_by_conversion_date": "DOUBLE",
+    "metrics.all_conversions_value_per_cost": "DOUBLE",
+    "metrics.all_new_customer_lifetime_value": "DOUBLE",
+    "metrics.asset_best_performance_cost_percentage": "DOUBLE",
+    "metrics.asset_best_performance_impression_percentage": "DOUBLE",
+    "metrics.asset_good_performance_cost_percentage": "DOUBLE",
+    "metrics.asset_good_performance_impression_percentage": "DOUBLE",
+    "metrics.asset_learning_performance_cost_percentage": "DOUBLE",
+    "metrics.asset_learning_performance_impression_percentage": "DOUBLE",
+    "metrics.asset_low_performance_cost_percentage": "DOUBLE",
+    "metrics.asset_low_performance_impression_percentage": "DOUBLE",
+    "metrics.asset_pinned_as_description_position_one_count": "INT64",
+    "metrics.asset_pinned_as_description_position_two_count": "INT64",
+    "metrics.asset_pinned_as_headline_position_one_count": "INT64",
+    "metrics.asset_pinned_as_headline_position_three_count": "INT64",
+    "metrics.asset_pinned_as_headline_position_two_count": "INT64",
+    "metrics.asset_pinned_total_count": "INT64",
+    "metrics.asset_unrated_performance_cost_percentage": "DOUBLE",
+    "metrics.asset_unrated_performance_impression_percentage": "DOUBLE",
+    "metrics.auction_insight_search_absolute_top_impression_percentage": "DOUBLE",
+    "metrics.auction_insight_search_impression_share": "DOUBLE",
+    "metrics.auction_insight_search_outranking_share": "DOUBLE",
+    "metrics.auction_insight_search_overlap_rate": "DOUBLE",
+    "metrics.auction_insight_search_position_above_rate": "DOUBLE",
+    "metrics.auction_insight_search_top_impression_percentage": "DOUBLE",
+    "metrics.average_cart_size": "DOUBLE",
+    "metrics.average_cost": "DOUBLE",
+    "metrics.average_cpc": "DOUBLE",
+    "metrics.average_cpe": "DOUBLE",
+    "metrics.average_cpm": "DOUBLE",
+    "metrics.average_cpv": "DOUBLE",
+    "metrics.average_impression_frequency_per_user": "DOUBLE",
+    "metrics.average_order_value_micros": "INT64",
+    "metrics.average_page_views": "DOUBLE",
+    "metrics.average_target_cpa_micros": "INT64",
+    "metrics.average_target_roas": "DOUBLE",
+    "metrics.average_time_on_site": "DOUBLE",
+    "metrics.benchmark_average_max_cpc": "DOUBLE",
+    "metrics.benchmark_ctr": "DOUBLE",
+    "metrics.biddable_app_install_conversions": "DOUBLE",
+    "metrics.biddable_app_post_install_conversions": "DOUBLE",
+    "metrics.bounce_rate": "DOUBLE",
+    "metrics.clicks": "INT64",
+    "metrics.combined_clicks": "INT64",
+    "metrics.combined_clicks_per_query": "DOUBLE",
+    "metrics.combined_queries": "INT64",
+    "metrics.content_budget_lost_impression_share": "DOUBLE",
+    "metrics.content_impression_share": "DOUBLE",
+    "metrics.content_rank_lost_impression_share": "DOUBLE",
+    "metrics.conversion_last_conversion_date": "DATE",
+    "metrics.conversion_last_received_request_date_time": "DATE",
+    "metrics.conversions": "DOUBLE",
+    "metrics.conversions_by_conversion_date": "DOUBLE",
+    "metrics.conversions_from_interactions_rate": "DOUBLE",
+    "metrics.conversions_from_interactions_value_per_interaction": "DOUBLE",
+    "metrics.conversions_value": "DOUBLE",
+    "metrics.conversions_value_by_conversion_date": "DOUBLE",
+    "metrics.conversions_value_per_cost": "DOUBLE",
+    "metrics.cost_micros": "INT64",
+    "metrics.cost_of_goods_sold_micros": "INT64",
+    "metrics.cost_per_all_conversions": "DOUBLE",
+    "metrics.cost_per_conversion": "DOUBLE",
+    "metrics.cost_per_current_model_attributed_conversion": "DOUBLE",
+    "metrics.cross_device_conversions": "DOUBLE",
+    "metrics.cross_device_conversions_value_micros": "INT64",
+    "metrics.cross_sell_cost_of_goods_sold_micros": "INT64",
+    "metrics.cross_sell_gross_profit_micros": "INT64",
+    "metrics.cross_sell_revenue_micros": "INT64",
+    "metrics.cross_sell_units_sold": "DOUBLE",
+    "metrics.ctr": "DOUBLE",
+    "metrics.current_model_attributed_conversions": "DOUBLE",
+    "metrics.current_model_attributed_conversions_from_interactions_rate": "DOUBLE",
+    "metrics.current_model_attributed_conversions_from_interactions_value_per_interaction": "DOUBLE",
+    "metrics.current_model_attributed_conversions_value": "DOUBLE",
+    "metrics.current_model_attributed_conversions_value_per_cost": "DOUBLE",
+    "metrics.eligible_impressions_from_location_asset_store_reach": "INT64",
+    "metrics.engagement_rate": "DOUBLE",
+    "metrics.engagements": "INT64",
+    "metrics.general_invalid_click_rate": "DOUBLE",
+    "metrics.general_invalid_clicks": "INT64",
+    "metrics.gmail_forwards": "INT64",
+    "metrics.gmail_saves": "INT64",
+    "metrics.gmail_secondary_clicks": "INT64",
+    "metrics.gross_profit_margin": "DOUBLE",
+    "metrics.gross_profit_micros": "INT64",
+    "metrics.historical_creative_quality_score": "ENUM",
+    "metrics.historical_landing_page_quality_score": "ENUM",
+    "metrics.historical_quality_score": "INT64",
+    "metrics.historical_search_predicted_ctr": "ENUM",
+    "metrics.hotel_average_lead_value_micros": "DOUBLE",
+    "metrics.hotel_commission_rate_micros": "INT64",
+    "metrics.hotel_eligible_impressions": "INT64",
+    "metrics.hotel_expected_commission_cost": "DOUBLE",
+    "metrics.hotel_price_difference_percentage": "DOUBLE",
+    "metrics.impressions": "INT64",
+    "metrics.impressions_from_store_reach": "INT64",
+    "metrics.interaction_event_types": "ENUM",
+    "metrics.interaction_rate": "DOUBLE",
+    "metrics.interactions": "INT64",
+    "metrics.invalid_click_rate": "DOUBLE",
+    "metrics.invalid_clicks": "INT64",
+    "metrics.lead_cost_of_goods_sold_micros": "INT64",
+    "metrics.lead_gross_profit_micros": "INT64",
+    "metrics.lead_revenue_micros": "INT64",
+    "metrics.lead_units_sold": "DOUBLE",
+    "metrics.linked_entities_count": "INT64",
+    "metrics.linked_sample_entities": "STRING",
+    "metrics.message_chat_rate": "DOUBLE",
+    "metrics.message_chats": "INT64",
+    "metrics.message_impressions": "INT64",
+    "metrics.mobile_friendly_clicks_percentage": "DOUBLE",
+    "metrics.new_customer_lifetime_value": "DOUBLE",
+    "metrics.optimization_score_uplift": "DOUBLE",
+    "metrics.optimization_score_url": "STRING",
+    "metrics.orders": "DOUBLE",
+    "metrics.organic_clicks": "INT64",
+    "metrics.organic_clicks_per_query": "DOUBLE",
+    "metrics.organic_impressions": "INT64",
+    "metrics.organic_impressions_per_query": "DOUBLE",
+    "metrics.organic_queries": "INT64",
+    "metrics.percent_new_visitors": "DOUBLE",
+    "metrics.phone_calls": "INT64",
+    "metrics.phone_impressions": "INT64",
+    "metrics.phone_through_rate": "DOUBLE",
+    "metrics.publisher_organic_clicks": "INT64",
+    "metrics.publisher_purchased_clicks": "INT64",
+    "metrics.publisher_unknown_clicks": "INT64",
+    "metrics.relative_ctr": "DOUBLE",
+    "metrics.results_conversions_purchase": "DOUBLE",
+    "metrics.revenue_micros": "INT64",
+    "metrics.sample_best_performance_entities": "STRING",
+    "metrics.sample_good_performance_entities": "STRING",
+    "metrics.sample_learning_performance_entities": "STRING",
+    "metrics.sample_low_performance_entities": "STRING",
+    "metrics.sample_unrated_performance_entities": "STRING",
+    "metrics.search_absolute_top_impression_share": "DOUBLE",
+    "metrics.search_budget_lost_absolute_top_impression_share": "DOUBLE",
+    "metrics.search_budget_lost_impression_share": "DOUBLE",
+    "metrics.search_budget_lost_top_impression_share": "DOUBLE",
+    "metrics.search_click_share": "DOUBLE",
+    "metrics.search_exact_match_impression_share": "DOUBLE",
+    "metrics.search_impression_share": "DOUBLE",
+    "metrics.search_rank_lost_absolute_top_impression_share": "DOUBLE",
+    "metrics.search_rank_lost_impression_share": "DOUBLE",
+    "metrics.search_rank_lost_top_impression_share": "DOUBLE",
+    "metrics.search_top_impression_share": "DOUBLE",
+    "metrics.search_volume": "MESSAGE",
+    "metrics.sk_ad_network_installs": "INT64",
+    "metrics.sk_ad_network_total_conversions": "INT64",
+    "metrics.speed_score": "INT64",
+    "metrics.store_visits_last_click_model_attributed_conversions": "DOUBLE",
+    "metrics.top_impression_percentage": "DOUBLE",
+    "metrics.unique_users": "INT64",
+    "metrics.units_sold": "DOUBLE",
+    "metrics.valid_accelerated_mobile_pages_clicks_percentage": "DOUBLE",
+    "metrics.value_per_all_conversions": "DOUBLE",
+    "metrics.value_per_all_conversions_by_conversion_date": "DOUBLE",
+    "metrics.value_per_conversion": "DOUBLE",
+    "metrics.value_per_conversions_by_conversion_date": "DOUBLE",
+    "metrics.value_per_current_model_attributed_conversion": "DOUBLE",
+    "metrics.video_quartile_p100_rate": "DOUBLE",
+    "metrics.video_quartile_p25_rate": "DOUBLE",
+    "metrics.video_quartile_p50_rate": "DOUBLE",
+    "metrics.video_quartile_p75_rate": "DOUBLE",
+    "metrics.video_view_rate": "DOUBLE",
+    "metrics.video_view_rate_in_feed": "DOUBLE",
+    "metrics.video_view_rate_in_stream": "DOUBLE",
+    "metrics.video_view_rate_shorts": "DOUBLE",
+    "metrics.video_views": "INT64",
+    "metrics.view_through_conversions": "INT64",
+    "metrics.view_through_conversions_from_location_asset_click_to_call": "DOUBLE",
+    "metrics.view_through_conversions_from_location_asset_directions": "DOUBLE",
+    "metrics.view_through_conversions_from_location_asset_menu": "DOUBLE",
+    "metrics.view_through_conversions_from_location_asset_order": "DOUBLE",
+    "metrics.view_through_conversions_from_location_asset_other_engagement": "DOUBLE",
+    "metrics.view_through_conversions_from_location_asset_store_visits": "DOUBLE",
+    "metrics.view_through_conversions_from_location_asset_website": "DOUBLE",
+}
+
+METRIC_TO_DLT_TYPE = {
+    "INT64": "bigint",
+    "DOUBLE": "double",
+    "STRING": "text",
+    "ENUM": "text",
+    # TODO: support message types
+    # "MESSAGE": "string",
+}
+
+
+def dlt_metrics_schema(metrics: List[str]):
+    """
+    Returns a dictionary with only the metrics that are
+    present in the given list of metrics.
+    """
+    schema = {}
+    for metric in metrics:
+        typ = METRICS_SCHEMA.get(metric)
+        if typ is None:
+            raise ValueError(f"Unsupported metric {metric}")
+
+        if typ not in METRIC_TO_DLT_TYPE:
+            raise ValueError(f"Unsupported metric '{metric}' of type '{typ}'")
+
+        # ???: can we make these non-nullable?
+        schema[field.to_column(metric)] = {
+            "data_type": METRIC_TO_DLT_TYPE[typ],
+        }
+    return schema

omniload/src/google_ads/predicates.py

@@ -0,0 +1,37 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import date, datetime, timezone
+from typing import Optional
+
+
+def date_predicate(column: str, start_date: date, end_date: Optional[date]) -> str:
+    """
+    Generates a date predicate for the WHERE clause of a
+    GAQL query.
+    """
+    if start_date is None:
+        raise ValueError("start_date must be provided")
+
+    if end_date is None:
+        end_date = datetime.now(tz=timezone.utc).date()
+
+    clauses = []
+    if start_date is not None:
+        clauses.append(f"""{column} >= '{start_date.strftime("%Y-%m-%d")}'""")
+
+    if end_date is not None:
+        clauses.append(f"""{column} <= '{end_date.strftime("%Y-%m-%d")}'""")
+
+    return " AND ".join(clauses)