ingestr 0.12.2__py3-none-any.whl → 0.12.4__py3-none-any.whl

ingestr/src/google_analytics/helpers/__init__.py

@@ -0,0 +1,70 @@
+"""Google analytics source helpers"""
+
+from typing import Iterator, List
+
+import dlt
+from apiclient.discovery import Resource  # type: ignore
+from dlt.common import logger, pendulum
+from dlt.common.typing import TDataItem
+from google.analytics.data_v1beta.types import (
+    Dimension,
+    Metric,
+)
+from pendulum.datetime import DateTime
+
+from .data_processing import get_report
+
+
+def basic_report(
+    client: Resource,
+    rows_per_page: int,
+    dimensions: List[str],
+    metrics: List[str],
+    property_id: int,
+    resource_name: str,
+    start_date: str,
+    last_date: dlt.sources.incremental[DateTime],
+) -> Iterator[TDataItem]:
+    """
+    Retrieves the data for a report given dimensions, metrics, and filters required for the report.
+
+    Args:
+        client: The Google Analytics client used to make requests.
+        dimensions: Dimensions for the report. See metadata for the full list of dimensions.
+        metrics: Metrics for the report. See metadata for the full list of metrics.
+        property_id: A reference to the Google Analytics project.
+            More info: https://developers.google.com/analytics/devguides/reporting/data/v1/property-id
+        rows_per_page: Controls how many rows are retrieved per page in the reports.
+            Default is 10000, maximum possible is 100000.
+        resource_name: The resource name used to save incremental into dlt state.
+        start_date: Incremental load start_date.
+            Default is taken from dlt state if it exists.
+        last_date: Incremental load end date.
+            Default is taken from dlt state if it exists.
+
+    Returns:
+        Generator of all rows of data in the report.
+    """
+
+    # grab the start time from last dlt load if not filled, if that is also empty then use the first day of the millennium as the start time instead
+    if last_date.last_value:
+        if start_date != "2015-08-14":
+            logger.warning(
+                f"Using the starting date: {last_date.last_value} for incremental report: {resource_name} and ignoring start date passed as argument {start_date}"
+            )
+        start_date = last_date.last_value.to_date_string()
+    else:
+        start_date = start_date or "2015-08-14"
+
+    processed_response = get_report(
+        client=client,
+        property_id=property_id,
+        # fill dimensions and metrics with the proper api client objects
+        dimension_list=[Dimension(name=dimension) for dimension in dimensions],
+        metric_list=[Metric(name=metric) for metric in metrics],
+        limit=rows_per_page,
+        start_date=start_date,
+        # configure end_date to yesterday as a date string
+        end_date=pendulum.now().to_date_string(),
+    )
+    yield from processed_response

ingestr/src/google_analytics/helpers/data_processing.py

@@ -0,0 +1,176 @@
+"""
+This module contains helpers that process data and make it ready for loading into the database
+"""
+
+import json
+from typing import Any, Iterator, List, Union
+
+import proto
+from dlt.common.exceptions import MissingDependencyException
+from dlt.common.pendulum import pendulum
+from dlt.common.typing import DictStrAny, TDataItem, TDataItems
+
+try:
+    from google.analytics.data_v1beta import BetaAnalyticsDataClient  # noqa: F401
+    from google.analytics.data_v1beta.types import (
+        DateRange,
+        Dimension,
+        DimensionExpression,  # noqa: F401
+        DimensionMetadata,  # noqa: F401
+        GetMetadataRequest,  # noqa: F401
+        Metadata,  # noqa: F401
+        Metric,
+        MetricMetadata,  # noqa: F401
+        MetricType,
+        RunReportRequest,
+        RunReportResponse,
+    )
+except ImportError:
+    raise MissingDependencyException(
+        "Google Analytics API Client", ["google-analytics-data"]
+    )
+try:
+    from apiclient.discovery import Resource, build  # type: ignore # noqa: F401
+except ImportError:
+    raise MissingDependencyException("Google API Client", ["google-api-python-client"])
+
+
+def to_dict(item: Any) -> Iterator[TDataItem]:
+    """
+    Processes a batch result (page of results per dimension) accordingly
+    :param batch:
+    :return:
+    """
+    item = json.loads(
+        proto.Message.to_json(
+            item,
+            preserving_proto_field_name=True,
+            use_integers_for_enums=False,
+            including_default_value_fields=False,
+        )
+    )
+    yield item
+
+
+def get_report(
+    client: Resource,
+    property_id: int,
+    dimension_list: List[Dimension],
+    metric_list: List[Metric],
+    limit: int,
+    start_date: str,
+    end_date: str,
+) -> Iterator[TDataItem]:
+    """
+    Gets all the possible pages of reports with the given query parameters.
+    Processes every page and yields a dictionary for every row of the report.
+
+    Args:
+        client: The Google Analytics client used to make requests.
+        property_id: A reference to the Google Analytics project.
+            More info: https://developers.google.com/analytics/devguides/reporting/data/v1/property-id
+        dimension_list: A list of all the dimensions requested in the query.
+        metric_list: A list of all the metrics requested in the query.
+        limit: Describes how many rows there should be per page.
+        start_date: The starting date of the query.
+        end_date: The ending date of the query.
+
+    Yields:
+        Generator of all rows of data in the report.
+    """
+
+    request = RunReportRequest(
+        property=f"properties/{property_id}",
+        dimensions=dimension_list,
+        metrics=metric_list,
+        limit=limit,
+        date_ranges=[DateRange(start_date=start_date, end_date=end_date)],
+    )
+    # process request
+    response = client.run_report(request)
+    processed_response_generator = process_report(response=response)
+    yield from processed_response_generator
+
+
+def process_report(response: RunReportResponse) -> Iterator[TDataItems]:
+    """
+    Receives a single page for a report response, processes it, and returns a generator for every row of data in the report page.
+
+    Args:
+        response: The API response for a single page of the report.
+
+    Yields:
+        Generator of dictionaries for every row of the report page.
+    """
+
+    metrics_headers = [header.name for header in response.metric_headers]
+    dimensions_headers = [header.name for header in response.dimension_headers]
+
+    distinct_key_combinations = {}
+
+    for row in response.rows:
+        response_dict: DictStrAny = {
+            dimension_header: _resolve_dimension_value(
+                dimension_header, dimension_value.value
+            )
+            for dimension_header, dimension_value in zip(
+                dimensions_headers, row.dimension_values
+            )
+        }
+
+        for i in range(len(metrics_headers)):
+            # get metric type and process the value depending on type. Save metric name including type as well for the columns
+            metric_type = response.metric_headers[i].type_
+            metric_value = process_metric_value(
+                metric_type=metric_type, value=row.metric_values[i].value
+            )
+            response_dict[metrics_headers[i]] = metric_value
+
+        unique_key = "-".join(list(response_dict.keys()))
+        if unique_key not in distinct_key_combinations:
+            distinct_key_combinations[unique_key] = True
+
+        yield response_dict
+
+
+def process_metric_value(metric_type: MetricType, value: str) -> Union[str, int, float]:
+    """
+    Processes the metric type, converts it from string to the correct type, and returns it.
+
+    Args:
+        metric_type: The type of the metric.
+        value: The value of the metric as a string.
+
+    Returns:
+        The given value converted to the correct data type.
+    """
+
+    # So far according to GA4 documentation these are the correct types: https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/MetricType
+    # 0 for strings, 1 for ints and 2-12 are different types of floating points.
+    if metric_type.value == 0:
+        return value
+    elif metric_type.value == 1:
+        return int(value)
+    else:
+        return float(value)
+
+
+def _resolve_dimension_value(dimension_name: str, dimension_value: str) -> Any:
+    """
+    Helper function that receives a dimension's name and value and converts it to a datetime object if needed.
+
+    Args:
+        dimension_name: Name of the dimension.
+        dimension_value: Value of the dimension.
+
+    Returns:
+        The value of the dimension with the correct data type.
+    """
+    if dimension_name == "date":
+        return pendulum.from_format(dimension_value, "YYYYMMDD", tz="UTC")
+    elif dimension_name == "dateHour":
+        return pendulum.from_format(dimension_value, "YYYYMMDDHH", tz="UTC")
+    elif dimension_name == "dateHourMinute":
+        return pendulum.from_format(dimension_value, "YYYYMMDDHHmm", tz="UTC")
+    else:
+        return dimension_value

ingestr/src/sources.py

@@ -3,17 +3,44 @@ import csv
 import json
 import os
 import re
-from datetime import date
-from typing import Any, Callable, Optional
+from datetime import date, datetime
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterator,
+    List,
+    Literal,
+    Optional,
+    Union,
+)
 from urllib.parse import ParseResult, parse_qs, quote, urlparse
 
 import dlt
 import pendulum
-from dlt.common.configuration.specs import AwsCredentials
+import sqlalchemy
+from dlt.common.configuration.specs import (
+    AwsCredentials,
+)
+from dlt.common.libs.sql_alchemy import (
+    Engine,
+    MetaData,
+)
 from dlt.common.time import ensure_pendulum_datetime
-from dlt.common.typing import TSecretStrValue
-from dlt.sources.credentials import ConnectionStringCredentials
+from dlt.common.typing import TDataItem, TSecretStrValue
+from dlt.extract import Incremental
+from dlt.sources.credentials import (
+    ConnectionStringCredentials,
+)
 from dlt.sources.sql_database import sql_table
+from dlt.sources.sql_database.helpers import TableLoader
+from dlt.sources.sql_database.schema_types import (
+    ReflectionLevel,
+    SelectAny,
+    Table,
+    TTypeAdapter,
+)
+from sqlalchemy import Column
 from sqlalchemy import types as sa
 from sqlalchemy.dialects import mysql
 
@@ -28,6 +55,8 @@ from ingestr.src.dynamodb import dynamodb
 from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
 from ingestr.src.filesystem import readers
 from ingestr.src.filters import table_adapter_exclude_columns
+from ingestr.src.github import github_reactions, github_repo_events, github_stargazers
+from ingestr.src.google_analytics import google_analytics
 from ingestr.src.google_sheets import google_spreadsheet
 from ingestr.src.gorgias import gorgias_source
 from ingestr.src.hubspot import hubspot
@@ -39,7 +68,7 @@ from ingestr.src.notion import notion_databases
 from ingestr.src.shopify import shopify_source
 from ingestr.src.slack import slack_source
 from ingestr.src.stripe_analytics import stripe_source
-from ingestr.src.table_definition import table_string_to_dataclass
+from ingestr.src.table_definition import TableDefinition, table_string_to_dataclass
 from ingestr.src.tiktok_ads import tiktok_source
 from ingestr.src.time import isotime
 from ingestr.src.zendesk import zendesk_chat, zendesk_support, zendesk_talk
@@ -48,6 +77,9 @@ from ingestr.src.zendesk.helpers.credentials import (
     ZendeskCredentialsToken,
 )
 
+TableBackend = Literal["sqlalchemy", "pyarrow", "pandas", "connectorx"]
+TQueryAdapter = Callable[[SelectAny, Table], SelectAny]
+
 
 class SqlSource:
     table_builder: Callable
@@ -59,16 +91,16 @@ class SqlSource:
         return False
 
     def dlt_source(self, uri: str, table: str, **kwargs):
-        table_fields = table_string_to_dataclass(table)
+        table_fields = TableDefinition(dataset="custom", table="custom")
+        if not table.startswith("query:"):
+            table_fields = table_string_to_dataclass(table)
 
         incremental = None
         if kwargs.get("incremental_key"):
             start_value = kwargs.get("interval_start")
             end_value = kwargs.get("interval_end")
-
             incremental = dlt.sources.incremental(
                 kwargs.get("incremental_key", ""),
-                # primary_key=(),
                 initial_value=start_value,
                 end_value=end_value,
             )
@@ -87,6 +119,111 @@ class SqlSource:
                     query = query.order_by(kwargs.get("incremental_key"))
                 return query
 
+        defer_table_reflect = False
+        sql_backend = kwargs.get("sql_backend", "sqlalchemy")
+        if table.startswith("query:"):
+            if kwargs.get("sql_limit"):
+                raise ValueError(
+                    "sql_limit is not supported for custom queries, please apply the limit in the query instead"
+                )
+
+            sql_backend = "sqlalchemy"
+            defer_table_reflect = True
+            query_value = table.split(":", 1)[1]
+
+            # this is a very hacky version of the table_rows function. it is built this way to go around the dlt's table loader.
+            # I didn't want to write a full fledged sqlalchemy source for now, and wanted to benefit from the existing stuff to begin with.
+            # this is by no means a production ready solution, but it works for now.
+            # the core idea behind this implementation is to create a mock table instance with the columns that are absolutely necessary for the incremental load to work.
+            # the table loader will then use the query adapter callback to apply the actual query and load the rows.
+            def table_rows(
+                engine: Engine,
+                table: Union[Table, str],
+                metadata: MetaData,
+                chunk_size: int,
+                backend: TableBackend,
+                incremental: Optional[Incremental[Any]] = None,
+                table_adapter_callback: Callable[[Table], None] = None,  # type: ignore
+                reflection_level: ReflectionLevel = "minimal",
+                backend_kwargs: Dict[str, Any] = None,  # type: ignore
+                type_adapter_callback: Optional[TTypeAdapter] = None,
+                included_columns: Optional[List[str]] = None,
+                query_adapter_callback: Optional[TQueryAdapter] = None,
+                resolve_foreign_keys: bool = False,
+            ) -> Iterator[TDataItem]:
+                hints = {  # type: ignore
+                    "columns": [],
+                }
+                cols = []  # type: ignore
+
+                if incremental:
+                    switchDict = {
+                        int: sa.INTEGER,
+                        datetime: sa.TIMESTAMP,
+                        date: sa.DATE,
+                        pendulum.Date: sa.DATE,
+                        pendulum.DateTime: sa.TIMESTAMP,
+                    }
+
+                    if incremental.last_value is not None:
+                        cols.append(
+                            Column(
+                                incremental.cursor_path,
+                                switchDict[type(incremental.last_value)],  # type: ignore
+                            )
+                        )
+                    else:
+                        cols.append(Column(incremental.cursor_path, sa.TIMESTAMP))  # type: ignore
+
+                table = Table(
+                    "query_result",
+                    metadata,
+                    *cols,
+                )
+
+                loader = TableLoader(
+                    engine,
+                    backend,
+                    table,
+                    hints["columns"],  # type: ignore
+                    incremental=incremental,
+                    chunk_size=chunk_size,
+                    query_adapter_callback=query_adapter_callback,
+                )
+                try:
+                    yield from loader.load_rows(backend_kwargs)
+                finally:
+                    if getattr(engine, "may_dispose_after_use", False):
+                        engine.dispose()
+
+            dlt.sources.sql_database.table_rows = table_rows
+
+            def query_adapter_callback(query, table, incremental=None, engine=None):
+                params = {}
+                if incremental:
+                    params["interval_start"] = (
+                        incremental.last_value
+                        if incremental.last_value is not None
+                        else datetime(year=1, month=1, day=1)
+                    )
+                    if incremental.end_value is not None:
+                        params["interval_end"] = incremental.end_value
+                else:
+                    if ":interval_start" in query_value:
+                        params["interval_start"] = (
+                            datetime.min
+                            if kwargs.get("interval_start") is None
+                            else kwargs.get("interval_start")
+                        )
+                    if ":interval_end" in query_value:
+                        params["interval_end"] = (
+                            datetime.max
+                            if kwargs.get("interval_end") is None
+                            else kwargs.get("interval_end")
+                        )
+
+                return sqlalchemy.text(query_value).bindparams(**params)
+
         def type_adapter_callback(sql_type):
             if isinstance(sql_type, mysql.SET):
                 return sa.JSON
@@ -97,7 +234,7 @@ class SqlSource:
             schema=table_fields.dataset,
             table=table_fields.table,
             incremental=incremental,
-            backend=kwargs.get("sql_backend", "sqlalchemy"),
+            backend=sql_backend,
             chunk_size=kwargs.get("page_size", None),
             reflection_level=reflection_level,
             query_adapter_callback=query_adapter_callback,
@@ -105,6 +242,7 @@ class SqlSource:
             table_adapter_callback=table_adapter_exclude_columns(
                 kwargs.get("sql_exclude_columns", [])
             ),
+            defer_table_reflect=defer_table_reflect,
         )
 
         return builder_res
@@ -1015,29 +1153,28 @@ class TikTokSource:
         if not access_token:
             raise ValueError("access_token is required to connect to TikTok")
 
-        time_zone = source_fields.get("time_zone", "UTC")
+        timezone = "UTC"
+        if source_fields.get("timezone") is not None:
+            timezone = source_fields.get("timezone")[0]  # type: ignore
 
-        advertiser_id = source_fields.get("advertiser_id")
-        if not advertiser_id:
-            raise ValueError("advertiser_id is required to connect to TikTok")
+        advertiser_ids = source_fields.get("advertiser_ids")
+        if not advertiser_ids:
+            raise ValueError("advertiser_ids is required to connect to TikTok")
 
-        start_date = pendulum.now().subtract(days=90).in_tz(time_zone[0])
-        end_date = ensure_pendulum_datetime(pendulum.now()).in_tz(time_zone[0])
+        advertiser_ids = advertiser_ids[0].replace(" ", "").split(",")
+
+        start_date = pendulum.now().subtract(days=30).in_tz(timezone)
+        end_date = ensure_pendulum_datetime(pendulum.now()).in_tz(timezone)
 
         interval_start = kwargs.get("interval_start")
         if interval_start is not None:
-            start_date = ensure_pendulum_datetime(interval_start).in_tz(time_zone[0])
+            start_date = ensure_pendulum_datetime(interval_start).in_tz(timezone)
 
         interval_end = kwargs.get("interval_end")
         if interval_end is not None:
-            end_date = ensure_pendulum_datetime(interval_end).in_tz(time_zone[0])
+            end_date = ensure_pendulum_datetime(interval_end).in_tz(timezone)
 
-        page_size = kwargs.get("page_size")
-        if page_size is not None and not isinstance(page_size, int):
-            page_size = int(page_size)
-
-        if page_size > 1000:
-            page_size = 1000
+        page_size = min(1000, kwargs.get("page_size", 1000))
 
         if table.startswith("custom:"):
             fields = table.split(":", 3)
@@ -1049,28 +1186,61 @@ class TikTokSource:
             dimensions = fields[1].replace(" ", "").split(",")
             if (
                 "campaign_id" not in dimensions
-                and "advertiser_id" not in dimensions
                 and "adgroup_id" not in dimensions
                 and "ad_id" not in dimensions
             ):
                 raise ValueError(
-                    "You must provide one ID dimension. Please use one ID dimension from the following options: [campaign_id, advertiser_id, adgroup_id, ad_id]"
+                    "TikTok API requires at least one ID dimension, please use one of the following dimensions: [campaign_id, adgroup_id, ad_id]"
                 )
 
+            if "advertiser_id" in dimensions:
+                dimensions.remove("advertiser_id")
+
             metrics = fields[2].replace(" ", "").split(",")
-            filters = []
+            filtering_param = False
+            filter_name = ""
+            filter_value = []
             if len(fields) == 4:
-                filters = fields[3].replace(" ", "").split(",")
+
+                def parse_filters(filters_raw: str) -> dict:
+                    # Parse filter string like "key1=value1,key2=value2,value3,value4"
+                    filters = {}
+                    current_key = None
+
+                    for item in filters_raw.split(","):
+                        if "=" in item:
+                            # Start of a new key-value pair
+                            key, value = item.split("=")
+                            filters[key] = [value]  # Always start with a list
+                            current_key = key
+                        elif current_key is not None:
+                            # Additional value for the current key
+                            filters[current_key].append(item)
+
+                    # Convert single-item lists to simple values
+                    return {k: v[0] if len(v) == 1 else v for k, v in filters.items()}
+
+                filtering_param = True
+                filters = parse_filters(fields[3])
+                if len(filters) > 1:
+                    raise ValueError(
+                        "Only one filter is allowed for TikTok custom reports"
+                    )
+                filter_name = list(filters.keys())[0]
+                filter_value = list(map(int, filters[list(filters.keys())[0]]))
+
         return tiktok_source(
             start_date=start_date,
             end_date=end_date,
             access_token=access_token[0],
-            advertiser_id=advertiser_id[0],
-            time_zone=time_zone[0],
+            advertiser_ids=advertiser_ids,
+            timezone=timezone,
             dimensions=dimensions,
             metrics=metrics,
-            filters=filters,
             page_size=page_size,
+            filter_name=filter_name,
+            filter_value=filter_value,
+            filtering_param=filtering_param,
         ).with_resources(endpoint)
 
 
@@ -1171,3 +1341,103 @@ class DynamoDBSource:
             )
 
         return dynamodb(table, creds, incremental)
+
+
+class GoogleAnalyticsSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parse_uri = urlparse(uri)
+        source_fields = parse_qs(parse_uri.query)
+        cred_path = source_fields.get("credentials_path")
+
+        if not cred_path:
+            raise ValueError("credentials_path is required to connect Google Analytics")
+        credentials = {}
+
+        with open(cred_path[0], "r") as f:
+            credentials = json.load(f)
+
+        property_id = source_fields.get("property_id")
+        if not property_id:
+            raise ValueError("property_id is required to connect to Google Analytics")
+
+        interval_start = kwargs.get("interval_start")
+        start_date = (
+            interval_start.strftime("%Y-%m-%d") if interval_start else "2015-08-14"
+        )
+
+        fields = table.split(":")
+        if len(fields) != 3:
+            raise ValueError(
+                "Invalid table format. Expected format: custom:<dimensions>:<metrics>"
+            )
+
+        dimensions = fields[1].replace(" ", "").split(",")
+
+        datetime = ""
+        for dimension_datetime in ["date", "dateHourMinute", "dateHour"]:
+            if dimension_datetime in dimensions:
+                datetime = dimension_datetime
+                break
+        else:
+            raise ValueError(
+                "You must provide at least one dimension: [dateHour, dateHourMinute, date]"
+            )
+
+        metrics = fields[2].replace(" ", "").split(",")
+        queries = [
+            {"resource_name": "custom", "dimensions": dimensions, "metrics": metrics}
+        ]
+
+        return google_analytics(
+            property_id=property_id[0],
+            start_date=start_date,
+            datetime=datetime,
+            queries=queries,
+            credentials=credentials,
+        ).with_resources("basic_report")
+
+
+class GitHubSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Github takes care of incrementality on its own, you should not provide incremental_key"
+            )
+        # github://?access_token=<access_token>&owner=<owner>&repo=<repo>
+        parsed_uri = urlparse(uri)
+        source_fields = parse_qs(parsed_uri.query)
+
+        owner = source_fields.get("owner", [None])[0]
+        if not owner:
+            raise ValueError(
+                "owner of the repository is required to connect with GitHub"
+            )
+
+        repo = source_fields.get("repo", [None])[0]
+        if not repo:
+            raise ValueError(
+                "repo variable is required to retrieve data for a specific repository from GitHub."
+            )
+
+        access_token = source_fields.get("access_token", [None])[0]
+        if not access_token and table not in ["repo_events"]:
+            raise ValueError("access_token is required to connect with GitHub")
+
+        if table in ["issues", "pull_requests"]:
+            return github_reactions(
+                owner=owner, name=repo, access_token=access_token
+            ).with_resources(table)
+        elif table == "repo_events":
+            return github_repo_events(owner=owner, name=repo, access_token=access_token)
+        elif table == "stargazers":
+            return github_stargazers(owner=owner, name=repo, access_token=access_token)
+        else:
+            raise ValueError(
+                f"Resource '{table}' is not supported for GitHub source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )