omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/google_analytics/__init__.py

@@ -0,0 +1,144 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Defines all the sources and resources needed for Google Analytics V4
+"""
+
+from typing import Iterator, List, Optional, Union
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.typing import DictStrAny, TDataItem
+from dlt.sources import DltResource
+from dlt.sources.credentials import GcpOAuthCredentials, GcpServiceAccountCredentials
+from google.analytics.data_v1beta import BetaAnalyticsDataClient
+from google.analytics.data_v1beta.types import (
+    Dimension,
+    Metric,
+    MinuteRange,
+)
+
+from .helpers import get_realtime_report, get_report
+
+
+@dlt.source(max_table_nesting=0)
+def google_analytics(
+    datetime_dimension: str,
+    credentials: Union[
+        GcpOAuthCredentials, GcpServiceAccountCredentials
+    ] = dlt.secrets.value,
+    property_ids: List[str] = dlt.config.value,
+    queries: List[DictStrAny] = dlt.config.value,
+    start_date: Optional[pendulum.DateTime] = pendulum.datetime(2024, 1, 1),
+    end_date: Optional[pendulum.DateTime] = None,
+    rows_per_page: int = 10000,
+    minute_range_objects: List[MinuteRange] | None = None,
+) -> List[DltResource]:
+    validated_property_ids = []
+    for pid in property_ids:
+        try:
+            int_pid = int(pid)
+        except ValueError:
+            raise ValueError(
+                f"{pid} is an invalid google property id. Please use a numeric id, and not your Measurement ID like G-7F1AE12JLR"
+            )
+        if int_pid == 0:
+            raise ValueError(
+                "Google Analytics property id is 0. Did you forget to configure it?"
+            )
+        validated_property_ids.append(int_pid)
+
+    if not rows_per_page:
+        raise ValueError("Rows per page cannot be 0")
+    # generate access token for credentials if we are using OAuth2.0
+    if isinstance(credentials, GcpOAuthCredentials):
+        credentials.auth("https://www.googleapis.com/auth/analytics.readonly")
+
+    # Build the service object for Google Analytics api.
+    client = BetaAnalyticsDataClient(credentials=credentials.to_native_credentials())
+    if len(queries) > 1:
+        raise ValueError(
+            "Google Analytics supports a single query ingestion at a time, please give only one query"
+        )
+    query = queries[0]
+
+    # always add "date" to dimensions so we are able to track the last day of a report
+    dimensions = query["dimensions"]
+
+    @dlt.resource(
+        name="custom",
+        merge_key=datetime_dimension,
+        write_disposition="merge",
+    )
+    def basic_report(
+        incremental=dlt.sources.incremental(
+            datetime_dimension,
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterator[TDataItem]:
+        start_date = incremental.last_value
+        end_date = incremental.end_value
+        if start_date is None:
+            start_date = pendulum.datetime(2024, 1, 1)
+        if end_date is None:
+            end_date = pendulum.yesterday()
+        for property_id in validated_property_ids:
+            yield from get_report(
+                client=client,
+                property_id=property_id,
+                dimension_list=[Dimension(name=dimension) for dimension in dimensions],
+                metric_list=[Metric(name=metric) for metric in query["metrics"]],
+                per_page=rows_per_page,
+                start_date=start_date,
+                end_date=end_date,
+            )
+
+    # real time report
+    @dlt.resource(
+        name="realtime",
+        merge_key="ingested_at",
+        write_disposition="merge",
+    )
+    def real_time_report() -> Iterator[TDataItem]:
+        for property_id in validated_property_ids:
+            yield from get_realtime_report(
+                client=client,
+                property_id=property_id,
+                dimension_list=[Dimension(name=dimension) for dimension in dimensions],
+                metric_list=[Metric(name=metric) for metric in query["metrics"]],
+                per_page=rows_per_page,
+                minute_range_objects=minute_range_objects,
+            )
+
+    # res = dlt.resource(
+    #     basic_report, name="basic_report", merge_key=datetime_dimension, write_disposition="merge"
+    # )(
+    #     client=client,
+    #     rows_per_page=rows_per_page,
+    #     property_id=property_id,
+    #     dimensions=dimensions,
+    #     metrics=query["metrics"],
+    #     resource_name=resource_name,
+    #     last_date=dlt.sources.incremental(
+    #         datetime_dimension,
+    #         initial_value=start_date,
+    #         end_value=end_date,
+    #     ),
+    # )
+
+    return [basic_report, real_time_report]

omniload/src/google_analytics/helpers.py

@@ -0,0 +1,312 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains helpers that process data and make it ready for loading into the database
+"""
+
+import base64
+import json
+from typing import Any, Iterator, List, Union
+from urllib.parse import parse_qs, urlparse
+
+import proto
+from dlt.common.exceptions import MissingDependencyException
+from dlt.common.pendulum import pendulum
+from dlt.common.typing import DictStrAny, TDataItem, TDataItems
+
+try:
+    from google.analytics.data_v1beta import BetaAnalyticsDataClient  # noqa: F401
+    from google.analytics.data_v1beta.types import (
+        DateRange,
+        Dimension,
+        DimensionExpression,  # noqa: F401
+        DimensionMetadata,  # noqa: F401
+        GetMetadataRequest,  # noqa: F401
+        Metadata,  # noqa: F401
+        Metric,
+        MetricMetadata,  # noqa: F401
+        MetricType,
+        MinuteRange,
+        RunRealtimeReportRequest,
+        RunReportRequest,
+        RunReportResponse,
+    )
+except ImportError:
+    raise MissingDependencyException(
+        "Google Analytics API Client", ["google-analytics-data"]
+    )
+try:
+    from apiclient.discovery import Resource, build  # type: ignore # noqa: F401
+except ImportError:
+    raise MissingDependencyException("Google API Client", ["google-api-python-client"])
+
+
+def to_dict(item: Any) -> Iterator[TDataItem]:
+    """
+    Processes a batch result (page of results per dimension) accordingly
+    :param batch:
+    :return:
+    """
+    item = json.loads(
+        proto.Message.to_json(
+            item,
+            preserving_proto_field_name=True,
+            use_integers_for_enums=False,
+            including_default_value_fields=False,
+        )
+    )
+    yield item
+
+
+def get_realtime_report(
+    client: Resource,
+    property_id: int,
+    dimension_list: List[Dimension],
+    metric_list: List[Metric],
+    per_page: int,
+    minute_range_objects: List[MinuteRange] | None = None,
+) -> Iterator[TDataItem]:
+    """
+    Gets all the possible pages of reports with the given query parameters.
+    Processes every page and yields a dictionary for every row of the report.
+
+    Args:
+        client: The Google Analytics client used to make requests.
+        property_id: A reference to the Google Analytics project.
+            More info: https://developers.google.com/analytics/devguides/reporting/data/v1/property-id
+        dimension_list: A list of all the dimensions requested in the query.
+        metric_list: A list of all the metrics requested in the query.
+        limit: Describes how many rows there should be per page.
+
+    Yields:
+        Generator of all rows of data in the report.
+    """
+    offset = 0
+    ingest_at = pendulum.now().to_date_string()
+
+    while True:
+        request = RunRealtimeReportRequest(
+            property=f"properties/{property_id}",
+            dimensions=dimension_list,
+            metrics=metric_list,
+            limit=per_page,
+            minute_ranges=minute_range_objects if minute_range_objects else None,
+        )
+        response = client.run_realtime_report(request)
+
+        # process request
+        processed_response_generator = process_report(
+            response=response, ingest_at=ingest_at
+        )
+        # import pdb; pdb.set_trace()
+        for row in processed_response_generator:
+            row["property_id"] = str(property_id)
+            yield row
+        offset += per_page
+        if len(response.rows) < per_page or offset > 1000000:
+            break
+
+
+def get_report(
+    client: Resource,
+    property_id: int,
+    dimension_list: List[Dimension],
+    metric_list: List[Metric],
+    per_page: int,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime,
+) -> Iterator[TDataItem]:
+    """
+    Gets all the possible pages of reports with the given query parameters.
+    Processes every page and yields a dictionary for every row of the report.
+
+    Args:
+        client: The Google Analytics client used to make requests.
+        property_id: A reference to the Google Analytics project.
+            More info: https://developers.google.com/analytics/devguides/reporting/data/v1/property-id
+        dimension_list: A list of all the dimensions requested in the query.
+        metric_list: A list of all the metrics requested in the query.
+        limit: Describes how many rows there should be per page.
+        start_date: The starting date of the query.
+        end_date: The ending date of the query.
+
+    Yields:
+        Generator of all rows of data in the report.
+    """
+
+    offset = 0
+    while True:
+        request = RunReportRequest(
+            property=f"properties/{property_id}",
+            dimensions=dimension_list,
+            metrics=metric_list,
+            limit=per_page,
+            offset=offset,
+            date_ranges=[
+                DateRange(
+                    start_date=start_date.to_date_string(),
+                    end_date=end_date.to_date_string(),
+                )
+            ],
+        )
+        response = client.run_report(request)
+
+        # process request
+        processed_response_generator = process_report(response=response)
+
+        # import pdb; pdb.set_trace()
+        for row in processed_response_generator:
+            row["property_id"] = str(property_id)
+            yield row
+        offset += per_page
+        if len(response.rows) < per_page or offset > 1000000:
+            break
+
+
+def process_report(
+    response: RunReportResponse, ingest_at: str | None = None
+) -> Iterator[TDataItems]:
+    metrics_headers = [header.name for header in response.metric_headers]
+    dimensions_headers = [header.name for header in response.dimension_headers]
+
+    distinct_key_combinations = {}
+
+    for row in response.rows:
+        response_dict: DictStrAny = {
+            dimension_header: _resolve_dimension_value(
+                dimension_header, dimension_value.value
+            )
+            for dimension_header, dimension_value in zip(
+                dimensions_headers, row.dimension_values
+            )
+        }
+
+        for i in range(len(metrics_headers)):
+            # get metric type and process the value depending on type. Save metric name including type as well for the columns
+            metric_type = response.metric_headers[i].type_
+            metric_value = process_metric_value(
+                metric_type=metric_type, value=row.metric_values[i].value
+            )
+            response_dict[metrics_headers[i]] = metric_value
+        if ingest_at is not None:
+            response_dict["ingested_at"] = ingest_at
+
+        unique_key = "-".join(list(response_dict.keys()))
+        if unique_key not in distinct_key_combinations:
+            distinct_key_combinations[unique_key] = True
+
+        yield response_dict
+
+
+def process_metric_value(metric_type: MetricType, value: str) -> Union[str, int, float]:
+    """
+    Processes the metric type, converts it from string to the correct type, and returns it.
+
+    Args:
+        metric_type: The type of the metric.
+        value: The value of the metric as a string.
+
+    Returns:
+        The given value converted to the correct data type.
+    """
+
+    # So far according to GA4 documentation these are the correct types: https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/MetricType
+    # 0 for strings, 1 for ints and 2-12 are different types of floating points.
+    if metric_type.value == 0:
+        return value
+    elif metric_type.value == 1:
+        return int(value)
+    else:
+        return float(value)
+
+
+def _resolve_dimension_value(dimension_name: str, dimension_value: str) -> Any:
+    if dimension_name == "date":
+        return pendulum.from_format(dimension_value, "YYYYMMDD", tz="UTC")
+    elif dimension_name == "dateHour":
+        return pendulum.from_format(dimension_value, "YYYYMMDDHH", tz="UTC")
+    elif dimension_name == "dateHourMinute":
+        return pendulum.from_format(dimension_value, "YYYYMMDDHHmm", tz="UTC")
+    else:
+        return dimension_value
+
+
+def convert_minutes_ranges_to_minute_range_objects(
+    minutes_ranges: str,
+) -> List[MinuteRange]:
+    minutes_ranges = minutes_ranges.strip()
+    minutes = minutes_ranges.replace(" ", "").split(",")
+    if minutes == "":
+        raise ValueError(
+            "Invalid input. Minutes range should be startminute-endminute format. For example: 1-2,5-6"
+        )
+
+    minute_range_objects = []
+    for min_range in minutes:
+        if "-" not in min_range:
+            raise ValueError(
+                "Invalid input. Minutes range should be startminute-endminute format. For example: 1-2,5-6"
+            )
+        parts = min_range.split("-")
+
+        if not parts[0].isdigit() or not parts[1].isdigit():
+            raise ValueError(
+                f"Invalid input '{min_range}'. Both start and end minutes must be digits. For example: 1-2,5-6"
+            )
+
+        end_minutes_ago = int(parts[0])
+        start_minutes_ago = int(parts[1])
+        minute_range_objects.append(
+            MinuteRange(
+                name=f"{end_minutes_ago}-{start_minutes_ago} minutes ago",
+                start_minutes_ago=start_minutes_ago,
+                end_minutes_ago=end_minutes_ago,
+            )
+        )
+
+    return minute_range_objects
+
+
+def parse_google_analytics_uri(uri: str):
+    parse_uri = urlparse(uri)
+    source_fields = parse_qs(parse_uri.query)
+    cred_path = source_fields.get("credentials_path")
+    cred_base64 = source_fields.get("credentials_base64")
+
+    if not cred_path and not cred_base64:
+        raise ValueError(
+            "credentials_path or credentials_base64 is required to connect Google Analytics"
+        )
+    credentials = {}
+    if cred_path:
+        with open(cred_path[0], "r") as f:
+            credentials = json.load(f)
+    elif cred_base64:
+        credentials = json.loads(base64.b64decode(cred_base64[0]).decode("utf-8"))
+
+    property_id = source_fields.get("property_id")
+    if not property_id:
+        raise ValueError("property_id is required to connect to Google Analytics")
+
+    if (not cred_path and not cred_base64) or (not property_id):
+        raise ValueError(
+            "credentials_path or credentials_base64 and property_id are required to connect Google Analytics"
+        )
+
+    property_ids = [pid.strip() for pid in property_id[0].split(",") if pid.strip()]
+    if not property_ids:
+        raise ValueError("property_id is required to connect to Google Analytics")
+
+    return {"credentials": credentials, "property_ids": property_ids}

omniload/src/google_sheets/README.md

@@ -0,0 +1,95 @@
+# Google Sheets
+
+## Prepare your data
+
+We recommend to to use [Named Ranges](link to gsheets) to indicate which data should be extracted from a particular spreadsheet and this is how this source
+will work by default - when called with without setting any other options. All the named ranges will be converted into tables named after them and stored in the
+destination.
+* You can let the spreadsheet users to add and remove tables by just adding/removing the ranges, you do not need to configure the pipeline again.
+* You can indicate exactly the fragments of interest and only this data will be retrieved so it is the fastest.
+* You can name database tables by changing the range names.
+
+If you are not happy with the workflow above, you can:
+* Disable it by setting `get_named_ranges` option to False
+* Enable retrieving all sheets/tabs with `get_sheets` option set to True
+* Pass a list of ranges as supported by Google Sheets in `range_names`
+
+Note that hidden columns will be extracted.
+
+> 💡 You can load data from many spreadsheets and also rename the tables to which data is loaded. This is standard part of `dlt`, see `load_with_table_rename_and_multiple_spreadsheets` demo in `google_sheets_pipeline.py`
+
+### Make sure your data has headers and is a proper table
+**First row of any extracted range should contain headers**. Please make sure:
+1. The header names are strings and are unique.
+2. That all the columns that you intend to extract have a header.
+3. That data starts exactly at the origin of the range - otherwise source will remove padding but it is a waste of resources!
+
+When source detects any problems with headers or table layout **it will issue a WARNING in the log** so it makes sense to run your pipeline script manually/locally and fix all the problems.
+1. Columns without headers will be removed and not extracted!
+2. Columns with headers that does not contain any data will be removed.
+2. If there's any problems with reading headers (ie. header is not string or is empty or not unique): **the headers row will be extracted as data** and automatic header names will be used.
+3. Empty rows are ignored
+4. `dlt` will normalize range names and headers into table and column names - so they may be different in the database than in google sheets. Prefer small cap names without special characters!
+
+### Data Types
+`dlt` normalizer will use first row of data to infer types and will try to coerce following rows - creating variant columns if that is not possible. This is a standard behavior.
+**date time** and **date** types are also recognized and this happens via additional metadata that is retrieved for the first row.
+
+## Passing the spreadsheet id/url and explicit range names
+You can use both url of your spreadsheet that you can copy from the browser ie.
+```
+https://docs.google.com/spreadsheets/d/1VTtCiYgxjAwcIw7UM1_BSaxC3rzIpr0HwXZwd2OlPD4/edit?usp=sharing
+```
+or spreadsheet id (which is a part of the url)
+```
+1VTtCiYgxjAwcIw7UM1_BSaxC3rzIpr0HwXZwd2OlPD4
+```
+typically you pass it directly to the `google_spreadsheet` function
+
+**passing ranges**
+
+You can pass explicit ranges to the `google_spreadsheet`:
+1. sheet names
+2. named ranges
+3. any range in Google Sheet format ie. **sheet 1!A1:B7**
+
+
+## The `spreadsheet_info` table
+This table is repopulated after every load and keeps the information on loaded ranges:
+* id and title of the spreadsheet
+* name of the range as passed to the source
+* string representation of the loaded range
+* range above in parsed representation
+
+## Running on Airflow (and some under the hood information)
+Internally, the source loads all the data immediately in the `google_spreadsheet` before execution of the pipeline in `run`. No matter how many ranges you request, we make just two calls to the API to retrieve data. This works very well with typical scripts that create a dlt source with `google_spreadsheet` and then run it with `pipeline.run`.
+
+In case of Airflow, the source is created and executed separately. In typical configuration where runner is a separate machine, **this will load data twice**.
+
+**Moreover, you should not use `scc` decomposition in our Airflow helper**. It will create an instance of the source for each requested range in order to run a task that corresponds to it! Following our [Airflow deployment guide](https://dlthub.com/docs/walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer#2-modify-dag-file), this is how you should use `tasks.add_run` on `PipelineTasksGroup`:
+```python
+@dag(
+    schedule_interval='@daily',
+    start_date=pendulum.datetime(2023, 2, 1),
+    catchup=False,
+    max_active_runs=1,
+    default_args=default_task_args
+)
+def get_named_ranges():
+    tasks = PipelineTasksGroup("get_named_ranges", use_data_folder=False, wipe_local_data=True)
+
+    # import your source from pipeline script
+    from google_sheets import google_spreadsheet
+
+    pipeline = dlt.pipeline(
+        pipeline_name="get_named_ranges",
+        dataset_name="named_ranges_data",
+        destination='bigquery',
+    )
+
+    # do not use decompose to run `google_spreadsheet` in single task
+    tasks.add_run(pipeline, google_spreadsheet("1HhWHjqouQnnCIZAFa2rL6vT91YRN8aIhts22SUUR580"), decompose="none", trigger_rule="all_done", retries=0, provide_context=True)
+```
+
+## Setup credentials
+[We recommend to use service account for any production deployments](https://dlthub.com/docs/dlt-ecosystem/verified-sources/google_sheets#google-sheets-api-authentication)