ingestr 0.12.3__py3-none-any.whl → 0.12.4__py3-none-any.whl

ingestr/main.py

@@ -32,7 +32,7 @@ DATE_FORMATS = [
 
 # https://dlthub.com/docs/dlt-ecosystem/file-formats/parquet#supported-destinations
 PARQUET_SUPPORTED_DESTINATIONS = [
-    "bigquery",
+    "athena" "bigquery",
     "duckdb",
     "snowflake",
     "databricks",
@@ -287,8 +287,14 @@ def ingest(
             envvar="SQL_EXCLUDE_COLUMNS",
         ),
     ] = [],  # type: ignore
+    columns: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            help="The column types to be used for the destination table in the format of 'column_name:column_type'",
+            envvar="COLUMNS",
+        ),
+    ] = None,  # type: ignore
 ):
-    # TODO(turtledev): can't we move this to the top of this file?
     import hashlib
     import tempfile
     from datetime import datetime
@@ -296,6 +302,7 @@ def ingest(
     import dlt
     import humanize
     import typer
+    from dlt.common.data_types import TDataType
     from dlt.common.destination import Destination
     from dlt.common.pipeline import LoadInfo
     from dlt.common.runtime.collector import Collector, LogCollector
@@ -345,7 +352,7 @@ def ingest(
             not in dlt_dest.capabilities().supported_loader_file_formats
         ):
             print(
-                f"[red]Loader file format {loader_file_format.value} is not supported by the destination.[/red]"
+                f"[red]Loader file format {loader_file_format.value} is not supported by the destination, available formats: {dlt_dest.capabilities().supported_loader_file_formats}.[/red]"
             )
             raise typer.Abort()
 
@@ -357,6 +364,23 @@ def ingest(
         else:
             executable(source)
 
+    def parse_columns(columns: list[str]) -> dict[str, TDataType]:
+        from typing import cast, get_args
+
+        possible_types = get_args(TDataType)
+
+        types: dict[str, TDataType] = {}
+        for column in columns:
+            for candidate in column.split(","):
+                column_name, column_type = candidate.split(":")
+                if column_type not in possible_types:
+                    print(
+                        f"[red]Column type '{column_type}' is not supported, supported types: {possible_types}.[/red]"
+                    )
+                    raise typer.Abort()
+                types[column_name] = cast(TDataType, column_type)
+        return types
+
     track(
         "command_triggered",
         {
@@ -399,12 +423,20 @@ def ingest(
         column_hints: dict[str, TColumnSchema] = {}
         original_incremental_strategy = incremental_strategy
 
+        if columns:
+            column_types = parse_columns(columns)
+            for column_name, column_type in column_types.items():
+                column_hints[column_name] = {"data_type": column_type}
+
         merge_key = None
         if incremental_strategy == IncrementalStrategy.delete_insert:
             merge_key = incremental_key
             incremental_strategy = IncrementalStrategy.merge
             if incremental_key:
-                column_hints[incremental_key] = {"merge_key": True}
+                if incremental_key not in column_hints:
+                    column_hints[incremental_key] = {}
+
+                column_hints[incremental_key]["merge_key"] = True
 
         m = hashlib.sha256()
         m.update(dest_table.encode("utf-8"))
@@ -491,6 +523,21 @@ def ingest(
         if factory.source_scheme == "sqlite":
             source_table = "main." + source_table.split(".")[-1]
 
+        if (
+            incremental_key
+            and incremental_key in column_hints
+            and "data_type" in column_hints[incremental_key]
+            and column_hints[incremental_key]["data_type"] == "date"
+        ):
+            # By default, ingestr treats the start and end dates as datetime objects. While this worked fine for many cases, if the
+            # incremental field is a date, the start and end dates cannot be compared to the incremental field, and the ingestion would fail.
+            # In order to eliminate this, we have introduced a new option to ingestr, --columns, which allows the user to specify the column types for the destination table.
+            # This way, ingestr will know the data type of the incremental field, and will be able to convert the start and end dates to the correct data type before running the ingestion.
+            if interval_start:
+                interval_start = interval_start.date()  # type: ignore
+            if interval_end:
+                interval_end = interval_end.date()  # type: ignore
+
         dlt_source = source.dlt_source(
             uri=source_uri,
             table=source_table,

ingestr/src/factory.py

@@ -24,6 +24,8 @@ from ingestr.src.sources import (
     ChessSource,
     DynamoDBSource,
     FacebookAdsSource,
+    GitHubSource,
+    GoogleAnalyticsSource,
     GoogleSheetsSource,
     GorgiasSource,
     HubspotSource,
@@ -102,6 +104,7 @@ class SourceDestinationFactory:
         "gsheets": GoogleSheetsSource,
         "shopify": ShopifySource,
         "gorgias": GorgiasSource,
+        "github": GitHubSource,
         "chess": ChessSource,
         "stripe": StripeAnalyticsSource,
         "facebookads": FacebookAdsSource,
@@ -118,6 +121,7 @@ class SourceDestinationFactory:
         "dynamodb": DynamoDBSource,
         "asana": AsanaSource,
         "tiktok": TikTokSource,
+        "googleanalytics": GoogleAnalyticsSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/github/__init__.py

@@ -0,0 +1,149 @@
+"""Source that load github issues, pull requests and reactions for a specific repository via customizable graphql query. Loads events incrementally."""
+
+import urllib.parse
+from typing import Iterator, Optional, Sequence
+
+import dlt
+from dlt.common.typing import TDataItems
+from dlt.sources import DltResource
+
+from .helpers import get_reactions_data, get_rest_pages, get_stargazers
+
+
+@dlt.source
+def github_reactions(
+    owner: str,
+    name: str,
+    access_token: str = dlt.secrets.value,
+    items_per_page: int = 100,
+    max_items: Optional[int] = None,
+) -> Sequence[DltResource]:
+    """Get reactions associated with issues, pull requests and comments in the repo `name` with owner `owner`.
+
+    This source uses graphql to retrieve all issues (`issues` resource) and pull requests (`pull requests` resource) with the associated reactions (up to 100),
+    comments (up to 100) and reactions to comments (also up to 100). Internally graphql is used to retrieve data. It is cost optimized and you are able to retrieve the
+    data for fairly large repos quickly and cheaply.
+    You can and should change the queries in `queries.py` to include for example additional fields or connections. The source can be hacked to add more resources for other
+    repository nodes easily.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic access token. Will be injected from secrets if not provided.
+        items_per_page (int, optional): How many issues/pull requests to get in single page. Defaults to 100.
+        max_items (int, optional): How many issues/pull requests to get in total. None means All.
+        max_item_age_seconds (float, optional): Do not get items older than this. Defaults to None. NOT IMPLEMENTED
+
+    Returns:
+        Sequence[DltResource]: Two DltResources: `issues` with issues and `pull_requests` with pull requests
+    """
+    return (
+        dlt.resource(
+            get_reactions_data(
+                "issues",
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="issues",
+            write_disposition="replace",
+        ),
+        dlt.resource(
+            get_reactions_data(
+                "pullRequests",
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="pull_requests",
+            write_disposition="replace",
+        ),
+    )
+
+
+@dlt.source(max_table_nesting=0)
+def github_repo_events(
+    owner: str, name: str, access_token: Optional[str] = None
+) -> DltResource:
+    """Gets events for repository `name` with owner `owner` incrementally.
+
+    This source contains a single resource `repo_events` that gets given repository's events and dispatches them to separate tables with names based on event type.
+    The data is loaded incrementally. Subsequent runs will get only new events and append them to tables.
+    Please note that Github allows only for 300 events to be retrieved for public repositories. You should get the events frequently for the active repos.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic or fine-grained access token. If not provided, calls are made anonymously
+
+    Returns:
+        DltSource: source with the `repo_events` resource
+
+    """
+
+    # use naming function in table name to generate separate tables for each event
+    @dlt.resource(primary_key="id", table_name=lambda i: i["type"])
+    def repo_events(
+        last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "created_at", initial_value="1970-01-01T00:00:00Z", last_value_func=max
+        ),
+    ) -> Iterator[TDataItems]:
+        repos_path = (
+            f"/repos/{urllib.parse.quote(owner)}/{urllib.parse.quote(name)}/events"
+        )
+
+        for page in get_rest_pages(access_token, repos_path + "?per_page=100"):
+            yield page
+
+            # stop requesting pages if the last element was already older than initial value
+            # note: incremental will skip those items anyway, we just do not want to use the api limits
+            if last_created_at.start_out_of_range:
+                print(
+                    f"Overlap with previous run created at {last_created_at.initial_value}"
+                )
+                break
+
+    return repo_events
+
+
+@dlt.source
+def github_stargazers(
+    owner: str,
+    name: str,
+    access_token: str = dlt.secrets.value,
+    items_per_page: int = 100,
+    max_items: Optional[int] = None,
+) -> Sequence[DltResource]:
+    """Get stargazers in the repo `name` with owner `owner`.
+
+    This source uses graphql to retrieve all stargazers with the associated starred date,
+    Internally graphql is used to retrieve data. It is cost optimized and you are able to retrieve the
+    data for fairly large repos quickly and cheaply.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic access token. Will be injected from secrets if not provided.
+        items_per_page (int, optional): How many issues/pull requests to get in single page. Defaults to 100.
+        max_items (int, optional): How many issues/pull requests to get in total. None means All.
+
+    Returns:
+        Sequence[DltResource]: One DltResource: `stargazers`
+    """
+    return (
+        dlt.resource(
+            get_stargazers(
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="stargazers",
+            write_disposition="replace",
+        ),
+    )

ingestr/src/github/helpers.py

@@ -0,0 +1,193 @@
+from typing import Iterator, List, Optional, Tuple
+
+from dlt.common.typing import DictStrAny, StrAny
+from dlt.common.utils import chunks
+from dlt.sources.helpers import requests
+
+from .queries import COMMENT_REACTIONS_QUERY, ISSUES_QUERY, RATE_LIMIT, STARGAZERS_QUERY
+from .settings import GRAPHQL_API_BASE_URL, REST_API_BASE_URL
+
+
+#
+# Shared
+#
+def _get_auth_header(access_token: Optional[str]) -> StrAny:
+    if access_token:
+        return {"Authorization": f"Bearer {access_token}"}
+    else:
+        # REST API works without access token (with high rate limits)
+        return {}
+
+
+#
+# Rest API helpers
+#
+def get_rest_pages(access_token: Optional[str], query: str) -> Iterator[List[StrAny]]:
+    def _request(page_url: str) -> requests.Response:
+        r = requests.get(page_url, headers=_get_auth_header(access_token))
+        print(
+            f"got page {page_url}, requests left: " + r.headers["x-ratelimit-remaining"]
+        )
+        return r
+
+    next_page_url = REST_API_BASE_URL + query
+    while True:
+        r: requests.Response = _request(next_page_url)
+        page_items = r.json()
+        if len(page_items) == 0:
+            break
+        yield page_items
+        if "next" not in r.links:
+            break
+        next_page_url = r.links["next"]["url"]
+
+
+#
+# GraphQL API helpers
+#
+def get_stargazers(
+    owner: str,
+    name: str,
+    access_token: str,
+    items_per_page: int,
+    max_items: Optional[int],
+) -> Iterator[Iterator[StrAny]]:
+    variables = {"owner": owner, "name": name, "items_per_page": items_per_page}
+    for page_items in _get_graphql_pages(
+        access_token, STARGAZERS_QUERY, variables, "stargazers", max_items
+    ):
+        yield map(
+            lambda item: {"starredAt": item["starredAt"], "user": item["node"]},
+            page_items,
+        )
+
+
+def get_reactions_data(
+    node_type: str,
+    owner: str,
+    name: str,
+    access_token: str,
+    items_per_page: int,
+    max_items: Optional[int],
+) -> Iterator[Iterator[StrAny]]:
+    variables = {
+        "owner": owner,
+        "name": name,
+        "issues_per_page": items_per_page,
+        "first_reactions": 100,
+        "first_comments": 100,
+        "node_type": node_type,
+    }
+    for page_items in _get_graphql_pages(
+        access_token, ISSUES_QUERY % node_type, variables, node_type, max_items
+    ):
+        # use reactionGroups to query for reactions to comments that have any reactions. reduces cost by 10-50x
+        reacted_comment_ids = {}
+        for item in page_items:
+            for comment in item["comments"]["nodes"]:
+                if any(group["createdAt"] for group in comment["reactionGroups"]):
+                    # print(f"for comment {comment['id']}: has reaction")
+                    reacted_comment_ids[comment["id"]] = comment
+                # if "reactionGroups" in comment:
+                comment.pop("reactionGroups", None)
+
+        # get comment reactions by querying comment nodes separately
+        comment_reactions = _get_comment_reaction(
+            list(reacted_comment_ids.keys()), access_token
+        )
+        # attach the reaction nodes where they should be
+        for comment in comment_reactions.values():
+            comment_id = comment["id"]
+            reacted_comment_ids[comment_id]["reactions"] = comment["reactions"]
+        yield map(_extract_nested_nodes, page_items)
+
+
+def _extract_top_connection(data: StrAny, node_type: str) -> StrAny:
+    assert (
+        isinstance(data, dict) and len(data) == 1
+    ), f"The data with list of {node_type} must be a dictionary and contain only one element"
+    data = next(iter(data.values()))
+    return data[node_type]  # type: ignore
+
+
+def _extract_nested_nodes(item: DictStrAny) -> DictStrAny:
+    """Recursively moves `nodes` and `totalCount` to reduce nesting."""
+    item["reactions_totalCount"] = item["reactions"].get("totalCount", 0)
+    item["reactions"] = item["reactions"]["nodes"]
+    comments = item["comments"]
+    item["comments_totalCount"] = item["comments"].get("totalCount", 0)
+    for comment in comments["nodes"]:
+        if "reactions" in comment:
+            comment["reactions_totalCount"] = comment["reactions"].get("totalCount", 0)
+            comment["reactions"] = comment["reactions"]["nodes"]
+    item["comments"] = comments["nodes"]
+    return item
+
+
+def _run_graphql_query(
+    access_token: str, query: str, variables: DictStrAny
+) -> Tuple[StrAny, StrAny]:
+    def _request() -> requests.Response:
+        r = requests.post(
+            GRAPHQL_API_BASE_URL,
+            json={"query": query, "variables": variables},
+            headers=_get_auth_header(access_token),
+        )
+        return r
+
+    data = _request().json()
+    if "errors" in data:
+        raise ValueError(data)
+    data = data["data"]
+    # pop rate limits
+    rate_limit = data.pop("rateLimit", {"cost": 0, "remaining": 0})
+    return data, rate_limit
+
+
+def _get_graphql_pages(
+    access_token: str, query: str, variables: DictStrAny, node_type: str, max_items: int
+) -> Iterator[List[DictStrAny]]:
+    items_count = 0
+    while True:
+        data, rate_limit = _run_graphql_query(access_token, query, variables)
+        top_connection = _extract_top_connection(data, node_type)
+        data_items = (
+            top_connection["nodes"]
+            if "nodes" in top_connection
+            else top_connection["edges"]
+        )
+        items_count += len(data_items)
+        print(
+            f'Got {len(data_items)}/{items_count} {node_type}s, query cost {rate_limit["cost"]}, remaining credits: {rate_limit["remaining"]}'
+        )
+        if data_items:
+            yield data_items
+        else:
+            return
+        # print(data["repository"][node_type]["pageInfo"]["endCursor"])
+        variables["page_after"] = _extract_top_connection(data, node_type)["pageInfo"][
+            "endCursor"
+        ]
+        if max_items and items_count >= max_items:
+            print(f"Max items limit reached: {items_count} >= {max_items}")
+            return
+
+
+def _get_comment_reaction(comment_ids: List[str], access_token: str) -> StrAny:
+    """Builds a query from a list of comment nodes and returns associated reactions."""
+    idx = 0
+    data: DictStrAny = {}
+    for page_chunk in chunks(comment_ids, 50):
+        subs = []
+        for comment_id in page_chunk:
+            subs.append(COMMENT_REACTIONS_QUERY % (idx, comment_id))
+            idx += 1
+        subs.append(RATE_LIMIT)
+        query = "{" + ",\n".join(subs) + "}"
+        # print(query)
+        page, rate_limit = _run_graphql_query(access_token, query, {})
+        print(
+            f'Got {len(page)} comments, query cost {rate_limit["cost"]}, remaining credits: {rate_limit["remaining"]}'
+        )
+        data.update(page)
+    return data

ingestr/src/github/queries.py

@@ -0,0 +1,115 @@
+RATE_LIMIT = """
+  rateLimit {
+    limit
+    cost
+    remaining
+    resetAt
+  }
+"""
+
+ISSUES_QUERY = """
+query($owner: String!, $name: String!, $issues_per_page: Int!, $first_reactions: Int!, $first_comments: Int!, $page_after: String) {
+  repository(owner: $owner, name: $name) {
+    %s(first: $issues_per_page, orderBy: {field: CREATED_AT, direction: DESC}, after: $page_after) {
+      totalCount
+      pageInfo {
+        endCursor
+        startCursor
+      }
+      nodes {
+        # id
+        number
+        url
+        title
+        body
+        author {login avatarUrl url}
+        authorAssociation
+        closed
+        closedAt
+        createdAt
+        state
+        updatedAt
+        reactions(first: $first_reactions) {
+          totalCount
+          nodes {
+            # id
+            user {login avatarUrl url}
+            content
+            createdAt
+          }
+        }
+        comments(first: $first_comments) {
+          totalCount
+          nodes {
+            id
+            url
+            body
+            author {avatarUrl login url}
+            authorAssociation
+            createdAt
+            reactionGroups {content createdAt}
+            # reactions(first: 0) {
+            #   totalCount
+            #   nodes {
+            #     # id
+            #     user {login avatarUrl url}
+            #     content
+            #     createdAt
+            #   }
+            # }
+          }
+        }
+      }
+    }
+  }
+  rateLimit {
+    limit
+    cost
+    remaining
+    resetAt
+  }
+}
+"""
+
+COMMENT_REACTIONS_QUERY = """
+node_%s: node(id:"%s") {
+     ... on IssueComment {
+      id
+      reactions(first: 100) {
+        totalCount
+        nodes {
+            user {login avatarUrl url}
+            content
+            createdAt
+          }
+      }
+    }
+  }
+"""
+
+STARGAZERS_QUERY = """
+query($owner: String!, $name: String!, $items_per_page: Int!, $page_after: String) {
+  repository(owner: $owner, name: $name) {
+    stargazers(first: $items_per_page, orderBy: {field: STARRED_AT, direction: DESC}, after: $page_after) {
+      pageInfo {
+        endCursor
+        startCursor
+      }
+      edges {
+        starredAt
+        node {
+          login
+          avatarUrl
+          url
+        }
+      }
+    }
+  }
+  rateLimit {
+    limit
+    cost
+    remaining
+    resetAt
+  }
+}
+"""

ingestr/src/github/settings.py

@@ -0,0 +1,10 @@
+"""Github source settings and constants."""
+
+START_DATE = "1970-01-01T00:00:00Z"
+
+# rest queries
+REST_API_BASE_URL = "https://api.github.com"
+REPO_EVENTS_PATH = "/repos/%s/%s/events"
+
+# graphql queries
+GRAPHQL_API_BASE_URL = "https://api.github.com/graphql"

ingestr/src/google_analytics/__init__.py

@@ -0,0 +1,70 @@
+"""
+Defines all the sources and resources needed for Google Analytics V4
+"""
+
+from typing import List, Optional, Union
+
+import dlt
+from dlt.common.typing import DictStrAny
+from dlt.sources import DltResource
+from dlt.sources.credentials import GcpOAuthCredentials, GcpServiceAccountCredentials
+from google.analytics.data_v1beta import BetaAnalyticsDataClient
+
+from .helpers import basic_report
+
+
+@dlt.source(max_table_nesting=0)
+def google_analytics(
+    datetime: str,
+    credentials: Union[
+        GcpOAuthCredentials, GcpServiceAccountCredentials
+    ] = dlt.secrets.value,
+    property_id: int = dlt.config.value,
+    queries: List[DictStrAny] = dlt.config.value,
+    start_date: Optional[str] = "2015-08-14",
+    rows_per_page: int = 10000,
+) -> List[DltResource]:
+    try:
+        property_id = int(property_id)
+    except ValueError:
+        raise ValueError(
+            f"{property_id} is an invalid google property id. Please use a numeric id, and not your Measurement ID like G-7F1AE12JLR"
+        )
+    if property_id == 0:
+        raise ValueError(
+            "Google Analytics property id is 0. Did you forget to configure it?"
+        )
+    if not rows_per_page:
+        raise ValueError("Rows per page cannot be 0")
+    # generate access token for credentials if we are using OAuth2.0
+    if isinstance(credentials, GcpOAuthCredentials):
+        credentials.auth("https://www.googleapis.com/auth/analytics.readonly")
+
+    # Build the service object for Google Analytics api.
+    client = BetaAnalyticsDataClient(credentials=credentials.to_native_credentials())
+    if len(queries) > 1:
+        raise ValueError(
+            "Google Analytics supports a single query ingestion at a time, please give only one query"
+        )
+    query = queries[0]
+
+    # always add "date" to dimensions so we are able to track the last day of a report
+    dimensions = query["dimensions"]
+    resource_name = query["resource_name"]
+
+    res = dlt.resource(
+        basic_report, name="basic_report", merge_key=datetime, write_disposition="merge"
+    )(
+        client=client,
+        rows_per_page=rows_per_page,
+        property_id=property_id,
+        dimensions=dimensions,
+        metrics=query["metrics"],
+        resource_name=resource_name,
+        start_date=start_date,
+        last_date=dlt.sources.incremental(
+            datetime
+        ),  # pass empty primary key to avoid unique checks, a primary key defined by the resource will be used
+    )
+
+    return [res]

ingestr/src/google_analytics/helpers/__init__.py

@@ -0,0 +1,70 @@
+"""Google analytics source helpers"""
+
+from typing import Iterator, List
+
+import dlt
+from apiclient.discovery import Resource  # type: ignore
+from dlt.common import logger, pendulum
+from dlt.common.typing import TDataItem
+from google.analytics.data_v1beta.types import (
+    Dimension,
+    Metric,
+)
+from pendulum.datetime import DateTime
+
+from .data_processing import get_report
+
+
+def basic_report(
+    client: Resource,
+    rows_per_page: int,
+    dimensions: List[str],
+    metrics: List[str],
+    property_id: int,
+    resource_name: str,
+    start_date: str,
+    last_date: dlt.sources.incremental[DateTime],
+) -> Iterator[TDataItem]:
+    """
+    Retrieves the data for a report given dimensions, metrics, and filters required for the report.
+
+    Args:
+        client: The Google Analytics client used to make requests.
+        dimensions: Dimensions for the report. See metadata for the full list of dimensions.
+        metrics: Metrics for the report. See metadata for the full list of metrics.
+        property_id: A reference to the Google Analytics project.
+            More info: https://developers.google.com/analytics/devguides/reporting/data/v1/property-id
+        rows_per_page: Controls how many rows are retrieved per page in the reports.
+            Default is 10000, maximum possible is 100000.
+        resource_name: The resource name used to save incremental into dlt state.
+        start_date: Incremental load start_date.
+            Default is taken from dlt state if it exists.
+        last_date: Incremental load end date.
+            Default is taken from dlt state if it exists.
+
+    Returns:
+        Generator of all rows of data in the report.
+    """
+
+    # grab the start time from last dlt load if not filled, if that is also empty then use the first day of the millennium as the start time instead
+    if last_date.last_value:
+        if start_date != "2015-08-14":
+            logger.warning(
+                f"Using the starting date: {last_date.last_value} for incremental report: {resource_name} and ignoring start date passed as argument {start_date}"
+            )
+        start_date = last_date.last_value.to_date_string()
+    else:
+        start_date = start_date or "2015-08-14"
+
+    processed_response = get_report(
+        client=client,
+        property_id=property_id,
+        # fill dimensions and metrics with the proper api client objects
+        dimension_list=[Dimension(name=dimension) for dimension in dimensions],
+        metric_list=[Metric(name=metric) for metric in metrics],
+        limit=rows_per_page,
+        start_date=start_date,
+        # configure end_date to yesterday as a date string
+        end_date=pendulum.now().to_date_string(),
+    )
+    yield from processed_response

ingestr/src/google_analytics/helpers/data_processing.py

@@ -0,0 +1,176 @@
+"""
+This module contains helpers that process data and make it ready for loading into the database
+"""
+
+import json
+from typing import Any, Iterator, List, Union
+
+import proto
+from dlt.common.exceptions import MissingDependencyException
+from dlt.common.pendulum import pendulum
+from dlt.common.typing import DictStrAny, TDataItem, TDataItems
+
+try:
+    from google.analytics.data_v1beta import BetaAnalyticsDataClient  # noqa: F401
+    from google.analytics.data_v1beta.types import (
+        DateRange,
+        Dimension,
+        DimensionExpression,  # noqa: F401
+        DimensionMetadata,  # noqa: F401
+        GetMetadataRequest,  # noqa: F401
+        Metadata,  # noqa: F401
+        Metric,
+        MetricMetadata,  # noqa: F401
+        MetricType,
+        RunReportRequest,
+        RunReportResponse,
+    )
+except ImportError:
+    raise MissingDependencyException(
+        "Google Analytics API Client", ["google-analytics-data"]
+    )
+try:
+    from apiclient.discovery import Resource, build  # type: ignore # noqa: F401
+except ImportError:
+    raise MissingDependencyException("Google API Client", ["google-api-python-client"])
+
+
+def to_dict(item: Any) -> Iterator[TDataItem]:
+    """
+    Processes a batch result (page of results per dimension) accordingly
+    :param batch:
+    :return:
+    """
+    item = json.loads(
+        proto.Message.to_json(
+            item,
+            preserving_proto_field_name=True,
+            use_integers_for_enums=False,
+            including_default_value_fields=False,
+        )
+    )
+    yield item
+
+
+def get_report(
+    client: Resource,
+    property_id: int,
+    dimension_list: List[Dimension],
+    metric_list: List[Metric],
+    limit: int,
+    start_date: str,
+    end_date: str,
+) -> Iterator[TDataItem]:
+    """
+    Gets all the possible pages of reports with the given query parameters.
+    Processes every page and yields a dictionary for every row of the report.
+
+    Args:
+        client: The Google Analytics client used to make requests.
+        property_id: A reference to the Google Analytics project.
+            More info: https://developers.google.com/analytics/devguides/reporting/data/v1/property-id
+        dimension_list: A list of all the dimensions requested in the query.
+        metric_list: A list of all the metrics requested in the query.
+        limit: Describes how many rows there should be per page.
+        start_date: The starting date of the query.
+        end_date: The ending date of the query.
+
+    Yields:
+        Generator of all rows of data in the report.
+    """
+
+    request = RunReportRequest(
+        property=f"properties/{property_id}",
+        dimensions=dimension_list,
+        metrics=metric_list,
+        limit=limit,
+        date_ranges=[DateRange(start_date=start_date, end_date=end_date)],
+    )
+    # process request
+    response = client.run_report(request)
+    processed_response_generator = process_report(response=response)
+    yield from processed_response_generator
+
+
+def process_report(response: RunReportResponse) -> Iterator[TDataItems]:
+    """
+    Receives a single page for a report response, processes it, and returns a generator for every row of data in the report page.
+
+    Args:
+        response: The API response for a single page of the report.
+
+    Yields:
+        Generator of dictionaries for every row of the report page.
+    """
+
+    metrics_headers = [header.name for header in response.metric_headers]
+    dimensions_headers = [header.name for header in response.dimension_headers]
+
+    distinct_key_combinations = {}
+
+    for row in response.rows:
+        response_dict: DictStrAny = {
+            dimension_header: _resolve_dimension_value(
+                dimension_header, dimension_value.value
+            )
+            for dimension_header, dimension_value in zip(
+                dimensions_headers, row.dimension_values
+            )
+        }
+
+        for i in range(len(metrics_headers)):
+            # get metric type and process the value depending on type. Save metric name including type as well for the columns
+            metric_type = response.metric_headers[i].type_
+            metric_value = process_metric_value(
+                metric_type=metric_type, value=row.metric_values[i].value
+            )
+            response_dict[metrics_headers[i]] = metric_value
+
+        unique_key = "-".join(list(response_dict.keys()))
+        if unique_key not in distinct_key_combinations:
+            distinct_key_combinations[unique_key] = True
+
+        yield response_dict
+
+
+def process_metric_value(metric_type: MetricType, value: str) -> Union[str, int, float]:
+    """
+    Processes the metric type, converts it from string to the correct type, and returns it.
+
+    Args:
+        metric_type: The type of the metric.
+        value: The value of the metric as a string.
+
+    Returns:
+        The given value converted to the correct data type.
+    """
+
+    # So far according to GA4 documentation these are the correct types: https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/MetricType
+    # 0 for strings, 1 for ints and 2-12 are different types of floating points.
+    if metric_type.value == 0:
+        return value
+    elif metric_type.value == 1:
+        return int(value)
+    else:
+        return float(value)
+
+
+def _resolve_dimension_value(dimension_name: str, dimension_value: str) -> Any:
+    """
+    Helper function that receives a dimension's name and value and converts it to a datetime object if needed.
+
+    Args:
+        dimension_name: Name of the dimension.
+        dimension_value: Value of the dimension.
+
+    Returns:
+        The value of the dimension with the correct data type.
+    """
+    if dimension_name == "date":
+        return pendulum.from_format(dimension_value, "YYYYMMDD", tz="UTC")
+    elif dimension_name == "dateHour":
+        return pendulum.from_format(dimension_value, "YYYYMMDDHH", tz="UTC")
+    elif dimension_name == "dateHourMinute":
+        return pendulum.from_format(dimension_value, "YYYYMMDDHHmm", tz="UTC")
+    else:
+        return dimension_value

ingestr/src/sources.py

@@ -29,7 +29,9 @@ from dlt.common.libs.sql_alchemy import (
 from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import TDataItem, TSecretStrValue
 from dlt.extract import Incremental
-from dlt.sources.credentials import ConnectionStringCredentials
+from dlt.sources.credentials import (
+    ConnectionStringCredentials,
+)
 from dlt.sources.sql_database import sql_table
 from dlt.sources.sql_database.helpers import TableLoader
 from dlt.sources.sql_database.schema_types import (
@@ -53,6 +55,8 @@ from ingestr.src.dynamodb import dynamodb
 from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
 from ingestr.src.filesystem import readers
 from ingestr.src.filters import table_adapter_exclude_columns
+from ingestr.src.github import github_reactions, github_repo_events, github_stargazers
+from ingestr.src.google_analytics import google_analytics
 from ingestr.src.google_sheets import google_spreadsheet
 from ingestr.src.gorgias import gorgias_source
 from ingestr.src.hubspot import hubspot
@@ -95,10 +99,8 @@ class SqlSource:
         if kwargs.get("incremental_key"):
             start_value = kwargs.get("interval_start")
             end_value = kwargs.get("interval_end")
-
             incremental = dlt.sources.incremental(
                 kwargs.get("incremental_key", ""),
-                # primary_key=(),
                 initial_value=start_value,
                 end_value=end_value,
             )
@@ -158,6 +160,7 @@ class SqlSource:
                     switchDict = {
                         int: sa.INTEGER,
                         datetime: sa.TIMESTAMP,
+                        date: sa.DATE,
                         pendulum.Date: sa.DATE,
                         pendulum.DateTime: sa.TIMESTAMP,
                     }
@@ -1338,3 +1341,103 @@ class DynamoDBSource:
             )
 
         return dynamodb(table, creds, incremental)
+
+
+class GoogleAnalyticsSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parse_uri = urlparse(uri)
+        source_fields = parse_qs(parse_uri.query)
+        cred_path = source_fields.get("credentials_path")
+
+        if not cred_path:
+            raise ValueError("credentials_path is required to connect Google Analytics")
+        credentials = {}
+
+        with open(cred_path[0], "r") as f:
+            credentials = json.load(f)
+
+        property_id = source_fields.get("property_id")
+        if not property_id:
+            raise ValueError("property_id is required to connect to Google Analytics")
+
+        interval_start = kwargs.get("interval_start")
+        start_date = (
+            interval_start.strftime("%Y-%m-%d") if interval_start else "2015-08-14"
+        )
+
+        fields = table.split(":")
+        if len(fields) != 3:
+            raise ValueError(
+                "Invalid table format. Expected format: custom:<dimensions>:<metrics>"
+            )
+
+        dimensions = fields[1].replace(" ", "").split(",")
+
+        datetime = ""
+        for dimension_datetime in ["date", "dateHourMinute", "dateHour"]:
+            if dimension_datetime in dimensions:
+                datetime = dimension_datetime
+                break
+        else:
+            raise ValueError(
+                "You must provide at least one dimension: [dateHour, dateHourMinute, date]"
+            )
+
+        metrics = fields[2].replace(" ", "").split(",")
+        queries = [
+            {"resource_name": "custom", "dimensions": dimensions, "metrics": metrics}
+        ]
+
+        return google_analytics(
+            property_id=property_id[0],
+            start_date=start_date,
+            datetime=datetime,
+            queries=queries,
+            credentials=credentials,
+        ).with_resources("basic_report")
+
+
+class GitHubSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Github takes care of incrementality on its own, you should not provide incremental_key"
+            )
+        # github://?access_token=<access_token>&owner=<owner>&repo=<repo>
+        parsed_uri = urlparse(uri)
+        source_fields = parse_qs(parsed_uri.query)
+
+        owner = source_fields.get("owner", [None])[0]
+        if not owner:
+            raise ValueError(
+                "owner of the repository is required to connect with GitHub"
+            )
+
+        repo = source_fields.get("repo", [None])[0]
+        if not repo:
+            raise ValueError(
+                "repo variable is required to retrieve data for a specific repository from GitHub."
+            )
+
+        access_token = source_fields.get("access_token", [None])[0]
+        if not access_token and table not in ["repo_events"]:
+            raise ValueError("access_token is required to connect with GitHub")
+
+        if table in ["issues", "pull_requests"]:
+            return github_reactions(
+                owner=owner, name=repo, access_token=access_token
+            ).with_resources(table)
+        elif table == "repo_events":
+            return github_repo_events(owner=owner, name=repo, access_token=access_token)
+        elif table == "stargazers":
+            return github_stargazers(owner=owner, name=repo, access_token=access_token)
+        else:
+            raise ValueError(
+                f"Resource '{table}' is not supported for GitHub source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.12.3"
+__version__ = "0.12.4"

{ingestr-0.12.3.dist-info → ingestr-0.12.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.3
+Version: 0.12.4
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -21,6 +21,7 @@ Requires-Dist: dlt==1.4.0
 Requires-Dist: duckdb-engine==0.13.5
 Requires-Dist: duckdb==1.1.3
 Requires-Dist: facebook-business==20.0.0
+Requires-Dist: google-analytics-data==0.18.15
 Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: mysql-connector-python==9.1.0

{ingestr-0.12.3.dist-info → ingestr-0.12.4.dist-info}/RECORD

@@ -1,12 +1,12 @@
-ingestr/main.py,sha256=wkU2uLMy1q8YarJ9mXNfJepeRjp6AuPDeNDOmMUt6n0,22309
+ingestr/main.py,sha256=AG6ycOEpCyBN1qEOzW3j8sKK8KX0mrBAL-A25MdRldY,24712
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=zcHJIIHAZmcD9sJomd6G1Bc-1KsxnBD2aByOSV_9L3g,8850
-ingestr/src/factory.py,sha256=UyE1TzTHn_V8JZno5SSYfQsho1eFYzzvOylogw4S49E,4389
+ingestr/src/factory.py,sha256=aE7TjHzONb4DKYcfh_6-CJJfvs4lmw7iUySvSm4yQbM,4516
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
-ingestr/src/sources.py,sha256=Jy1N5EfbxfTae0L7PiZmPVxVYWLvOuLlw3kJ6vbT50M,48027
+ingestr/src/sources.py,sha256=zkK24y3jyucbrW2MU3i0Rx1SImZWatM9_A_8Wa7ExCM,51887
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
-ingestr/src/version.py,sha256=Qu8-91hLcRe7wfW37PwNdivTonHHKLrqtJPOAq3Jvhc,23
+ingestr/src/version.py,sha256=DoMS9KOhsApLyuLYhLEsd5nmoLFQ_IvVkEs_jKRzFk8,23
 ingestr/src/adjust/__init__.py,sha256=NaRNwDhItG8Q7vUHw7zQvyfWjmT32M0CSc5ufjmBM9U,3067
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -27,6 +27,13 @@ ingestr/src/facebook_ads/settings.py,sha256=1IxZeP_4rN3IBvAncNHOoqpzAirx0Hz-MUK_
 ingestr/src/filesystem/__init__.py,sha256=wHHaKFuAjsR_ZRjl6g_Flf6FhVs9qhwREthTr03_7cc,4162
 ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
 ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
+ingestr/src/github/__init__.py,sha256=csA2VcjOxXrVrvp7zY-JodO9Lpy98bJ4AqRdHCLTcGM,5838
+ingestr/src/github/helpers.py,sha256=Tmnik9811zBWNO6cJwV9PFQxEx2j32LHAQCvNbubsEI,6759
+ingestr/src/github/queries.py,sha256=W34C02jUEdjFmOE7f7u9xvYyBNDMfVZAu0JIRZI2mkU,2302
+ingestr/src/github/settings.py,sha256=N5ahWrDIQ_4IWV9i-hTXxyYduqY9Ym2BTwqsWxcDdJ8,258
+ingestr/src/google_analytics/__init__.py,sha256=HjA13wfJm2MGfy3h_DiM5ekkNqM2dgwYCKJ3pprnDtI,2482
+ingestr/src/google_analytics/helpers/__init__.py,sha256=y_q7dinlEwNBEpq6kCzjTa8lAhe2bb23bDPP0fcy7fY,2744
+ingestr/src/google_analytics/helpers/data_processing.py,sha256=fIdEKr9CmZN_s1T2i9BL8IYTPPqNoK6Vaquq2y8StfE,6072
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
@@ -77,8 +84,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.12.3.dist-info/METADATA,sha256=Kh5v7a3mzxqmekWFp8ebU-uHKwzHRrN0HkNHPRV3_5U,7910
-ingestr-0.12.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.12.3.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.12.3.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.12.3.dist-info/RECORD,,
+ingestr-0.12.4.dist-info/METADATA,sha256=VN9cqnH_rmALlSxePi6XOxOxndDGLYWTW0K6eafYVDw,7956
+ingestr-0.12.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.12.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.12.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.12.4.dist-info/RECORD,,