ingestr 0.12.2__py3-none-any.whl → 0.12.4__py3-none-any.whl

ingestr/main.py

@@ -32,7 +32,7 @@ DATE_FORMATS = [
 
 # https://dlthub.com/docs/dlt-ecosystem/file-formats/parquet#supported-destinations
 PARQUET_SUPPORTED_DESTINATIONS = [
-    "bigquery",
+    "athena" "bigquery",
     "duckdb",
     "snowflake",
     "databricks",
@@ -287,8 +287,14 @@ def ingest(
             envvar="SQL_EXCLUDE_COLUMNS",
         ),
     ] = [],  # type: ignore
+    columns: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            help="The column types to be used for the destination table in the format of 'column_name:column_type'",
+            envvar="COLUMNS",
+        ),
+    ] = None,  # type: ignore
 ):
-    # TODO(turtledev): can't we move this to the top of this file?
     import hashlib
     import tempfile
     from datetime import datetime
@@ -296,6 +302,7 @@ def ingest(
     import dlt
     import humanize
     import typer
+    from dlt.common.data_types import TDataType
     from dlt.common.destination import Destination
     from dlt.common.pipeline import LoadInfo
     from dlt.common.runtime.collector import Collector, LogCollector
@@ -345,7 +352,7 @@ def ingest(
             not in dlt_dest.capabilities().supported_loader_file_formats
         ):
             print(
-                f"[red]Loader file format {loader_file_format.value} is not supported by the destination.[/red]"
+                f"[red]Loader file format {loader_file_format.value} is not supported by the destination, available formats: {dlt_dest.capabilities().supported_loader_file_formats}.[/red]"
             )
             raise typer.Abort()
 
@@ -357,6 +364,23 @@ def ingest(
         else:
             executable(source)
 
+    def parse_columns(columns: list[str]) -> dict[str, TDataType]:
+        from typing import cast, get_args
+
+        possible_types = get_args(TDataType)
+
+        types: dict[str, TDataType] = {}
+        for column in columns:
+            for candidate in column.split(","):
+                column_name, column_type = candidate.split(":")
+                if column_type not in possible_types:
+                    print(
+                        f"[red]Column type '{column_type}' is not supported, supported types: {possible_types}.[/red]"
+                    )
+                    raise typer.Abort()
+                types[column_name] = cast(TDataType, column_type)
+        return types
+
     track(
         "command_triggered",
         {
@@ -399,12 +423,20 @@ def ingest(
         column_hints: dict[str, TColumnSchema] = {}
         original_incremental_strategy = incremental_strategy
 
+        if columns:
+            column_types = parse_columns(columns)
+            for column_name, column_type in column_types.items():
+                column_hints[column_name] = {"data_type": column_type}
+
         merge_key = None
         if incremental_strategy == IncrementalStrategy.delete_insert:
             merge_key = incremental_key
             incremental_strategy = IncrementalStrategy.merge
             if incremental_key:
-                column_hints[incremental_key] = {"merge_key": True}
+                if incremental_key not in column_hints:
+                    column_hints[incremental_key] = {}
+
+                column_hints[incremental_key]["merge_key"] = True
 
         m = hashlib.sha256()
         m.update(dest_table.encode("utf-8"))
@@ -491,6 +523,21 @@ def ingest(
         if factory.source_scheme == "sqlite":
             source_table = "main." + source_table.split(".")[-1]
 
+        if (
+            incremental_key
+            and incremental_key in column_hints
+            and "data_type" in column_hints[incremental_key]
+            and column_hints[incremental_key]["data_type"] == "date"
+        ):
+            # By default, ingestr treats the start and end dates as datetime objects. While this worked fine for many cases, if the
+            # incremental field is a date, the start and end dates cannot be compared to the incremental field, and the ingestion would fail.
+            # In order to eliminate this, we have introduced a new option to ingestr, --columns, which allows the user to specify the column types for the destination table.
+            # This way, ingestr will know the data type of the incremental field, and will be able to convert the start and end dates to the correct data type before running the ingestion.
+            if interval_start:
+                interval_start = interval_start.date()  # type: ignore
+            if interval_end:
+                interval_end = interval_end.date()  # type: ignore
+
         dlt_source = source.dlt_source(
             uri=source_uri,
             table=source_table,

ingestr/src/factory.py

@@ -24,6 +24,8 @@ from ingestr.src.sources import (
     ChessSource,
     DynamoDBSource,
     FacebookAdsSource,
+    GitHubSource,
+    GoogleAnalyticsSource,
     GoogleSheetsSource,
     GorgiasSource,
     HubspotSource,
@@ -102,6 +104,7 @@ class SourceDestinationFactory:
         "gsheets": GoogleSheetsSource,
         "shopify": ShopifySource,
         "gorgias": GorgiasSource,
+        "github": GitHubSource,
         "chess": ChessSource,
         "stripe": StripeAnalyticsSource,
         "facebookads": FacebookAdsSource,
@@ -118,6 +121,7 @@ class SourceDestinationFactory:
         "dynamodb": DynamoDBSource,
         "asana": AsanaSource,
         "tiktok": TikTokSource,
+        "googleanalytics": GoogleAnalyticsSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/github/__init__.py

@@ -0,0 +1,149 @@
+"""Source that load github issues, pull requests and reactions for a specific repository via customizable graphql query. Loads events incrementally."""
+
+import urllib.parse
+from typing import Iterator, Optional, Sequence
+
+import dlt
+from dlt.common.typing import TDataItems
+from dlt.sources import DltResource
+
+from .helpers import get_reactions_data, get_rest_pages, get_stargazers
+
+
+@dlt.source
+def github_reactions(
+    owner: str,
+    name: str,
+    access_token: str = dlt.secrets.value,
+    items_per_page: int = 100,
+    max_items: Optional[int] = None,
+) -> Sequence[DltResource]:
+    """Get reactions associated with issues, pull requests and comments in the repo `name` with owner `owner`.
+
+    This source uses graphql to retrieve all issues (`issues` resource) and pull requests (`pull requests` resource) with the associated reactions (up to 100),
+    comments (up to 100) and reactions to comments (also up to 100). Internally graphql is used to retrieve data. It is cost optimized and you are able to retrieve the
+    data for fairly large repos quickly and cheaply.
+    You can and should change the queries in `queries.py` to include for example additional fields or connections. The source can be hacked to add more resources for other
+    repository nodes easily.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic access token. Will be injected from secrets if not provided.
+        items_per_page (int, optional): How many issues/pull requests to get in single page. Defaults to 100.
+        max_items (int, optional): How many issues/pull requests to get in total. None means All.
+        max_item_age_seconds (float, optional): Do not get items older than this. Defaults to None. NOT IMPLEMENTED
+
+    Returns:
+        Sequence[DltResource]: Two DltResources: `issues` with issues and `pull_requests` with pull requests
+    """
+    return (
+        dlt.resource(
+            get_reactions_data(
+                "issues",
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="issues",
+            write_disposition="replace",
+        ),
+        dlt.resource(
+            get_reactions_data(
+                "pullRequests",
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="pull_requests",
+            write_disposition="replace",
+        ),
+    )
+
+
+@dlt.source(max_table_nesting=0)
+def github_repo_events(
+    owner: str, name: str, access_token: Optional[str] = None
+) -> DltResource:
+    """Gets events for repository `name` with owner `owner` incrementally.
+
+    This source contains a single resource `repo_events` that gets given repository's events and dispatches them to separate tables with names based on event type.
+    The data is loaded incrementally. Subsequent runs will get only new events and append them to tables.
+    Please note that Github allows only for 300 events to be retrieved for public repositories. You should get the events frequently for the active repos.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic or fine-grained access token. If not provided, calls are made anonymously
+
+    Returns:
+        DltSource: source with the `repo_events` resource
+
+    """
+
+    # use naming function in table name to generate separate tables for each event
+    @dlt.resource(primary_key="id", table_name=lambda i: i["type"])
+    def repo_events(
+        last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "created_at", initial_value="1970-01-01T00:00:00Z", last_value_func=max
+        ),
+    ) -> Iterator[TDataItems]:
+        repos_path = (
+            f"/repos/{urllib.parse.quote(owner)}/{urllib.parse.quote(name)}/events"
+        )
+
+        for page in get_rest_pages(access_token, repos_path + "?per_page=100"):
+            yield page
+
+            # stop requesting pages if the last element was already older than initial value
+            # note: incremental will skip those items anyway, we just do not want to use the api limits
+            if last_created_at.start_out_of_range:
+                print(
+                    f"Overlap with previous run created at {last_created_at.initial_value}"
+                )
+                break
+
+    return repo_events
+
+
+@dlt.source
+def github_stargazers(
+    owner: str,
+    name: str,
+    access_token: str = dlt.secrets.value,
+    items_per_page: int = 100,
+    max_items: Optional[int] = None,
+) -> Sequence[DltResource]:
+    """Get stargazers in the repo `name` with owner `owner`.
+
+    This source uses graphql to retrieve all stargazers with the associated starred date,
+    Internally graphql is used to retrieve data. It is cost optimized and you are able to retrieve the
+    data for fairly large repos quickly and cheaply.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic access token. Will be injected from secrets if not provided.
+        items_per_page (int, optional): How many issues/pull requests to get in single page. Defaults to 100.
+        max_items (int, optional): How many issues/pull requests to get in total. None means All.
+
+    Returns:
+        Sequence[DltResource]: One DltResource: `stargazers`
+    """
+    return (
+        dlt.resource(
+            get_stargazers(
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="stargazers",
+            write_disposition="replace",
+        ),
+    )

ingestr/src/github/helpers.py

@@ -0,0 +1,193 @@
+from typing import Iterator, List, Optional, Tuple
+
+from dlt.common.typing import DictStrAny, StrAny
+from dlt.common.utils import chunks
+from dlt.sources.helpers import requests
+
+from .queries import COMMENT_REACTIONS_QUERY, ISSUES_QUERY, RATE_LIMIT, STARGAZERS_QUERY
+from .settings import GRAPHQL_API_BASE_URL, REST_API_BASE_URL
+
+
+#
+# Shared
+#
+def _get_auth_header(access_token: Optional[str]) -> StrAny:
+    if access_token:
+        return {"Authorization": f"Bearer {access_token}"}
+    else:
+        # REST API works without access token (with high rate limits)
+        return {}
+
+
+#
+# Rest API helpers
+#
+def get_rest_pages(access_token: Optional[str], query: str) -> Iterator[List[StrAny]]:
+    def _request(page_url: str) -> requests.Response:
+        r = requests.get(page_url, headers=_get_auth_header(access_token))
+        print(
+            f"got page {page_url}, requests left: " + r.headers["x-ratelimit-remaining"]
+        )
+        return r
+
+    next_page_url = REST_API_BASE_URL + query
+    while True:
+        r: requests.Response = _request(next_page_url)
+        page_items = r.json()
+        if len(page_items) == 0:
+            break
+        yield page_items
+        if "next" not in r.links:
+            break
+        next_page_url = r.links["next"]["url"]
+
+
+#
+# GraphQL API helpers
+#
+def get_stargazers(
+    owner: str,
+    name: str,
+    access_token: str,
+    items_per_page: int,
+    max_items: Optional[int],
+) -> Iterator[Iterator[StrAny]]:
+    variables = {"owner": owner, "name": name, "items_per_page": items_per_page}
+    for page_items in _get_graphql_pages(
+        access_token, STARGAZERS_QUERY, variables, "stargazers", max_items
+    ):
+        yield map(
+            lambda item: {"starredAt": item["starredAt"], "user": item["node"]},
+            page_items,
+        )
+
+
+def get_reactions_data(
+    node_type: str,
+    owner: str,
+    name: str,
+    access_token: str,
+    items_per_page: int,
+    max_items: Optional[int],
+) -> Iterator[Iterator[StrAny]]:
+    variables = {
+        "owner": owner,
+        "name": name,
+        "issues_per_page": items_per_page,
+        "first_reactions": 100,
+        "first_comments": 100,
+        "node_type": node_type,
+    }
+    for page_items in _get_graphql_pages(
+        access_token, ISSUES_QUERY % node_type, variables, node_type, max_items
+    ):
+        # use reactionGroups to query for reactions to comments that have any reactions. reduces cost by 10-50x
+        reacted_comment_ids = {}
+        for item in page_items:
+            for comment in item["comments"]["nodes"]:
+                if any(group["createdAt"] for group in comment["reactionGroups"]):
+                    # print(f"for comment {comment['id']}: has reaction")
+                    reacted_comment_ids[comment["id"]] = comment
+                # if "reactionGroups" in comment:
+                comment.pop("reactionGroups", None)
+
+        # get comment reactions by querying comment nodes separately
+        comment_reactions = _get_comment_reaction(
+            list(reacted_comment_ids.keys()), access_token
+        )
+        # attach the reaction nodes where they should be
+        for comment in comment_reactions.values():
+            comment_id = comment["id"]
+            reacted_comment_ids[comment_id]["reactions"] = comment["reactions"]
+        yield map(_extract_nested_nodes, page_items)
+
+
+def _extract_top_connection(data: StrAny, node_type: str) -> StrAny:
+    assert (
+        isinstance(data, dict) and len(data) == 1
+    ), f"The data with list of {node_type} must be a dictionary and contain only one element"
+    data = next(iter(data.values()))
+    return data[node_type]  # type: ignore
+
+
+def _extract_nested_nodes(item: DictStrAny) -> DictStrAny:
+    """Recursively moves `nodes` and `totalCount` to reduce nesting."""
+    item["reactions_totalCount"] = item["reactions"].get("totalCount", 0)
+    item["reactions"] = item["reactions"]["nodes"]
+    comments = item["comments"]
+    item["comments_totalCount"] = item["comments"].get("totalCount", 0)
+    for comment in comments["nodes"]:
+        if "reactions" in comment:
+            comment["reactions_totalCount"] = comment["reactions"].get("totalCount", 0)
+            comment["reactions"] = comment["reactions"]["nodes"]
+    item["comments"] = comments["nodes"]
+    return item
+
+
+def _run_graphql_query(
+    access_token: str, query: str, variables: DictStrAny
+) -> Tuple[StrAny, StrAny]:
+    def _request() -> requests.Response:
+        r = requests.post(
+            GRAPHQL_API_BASE_URL,
+            json={"query": query, "variables": variables},
+            headers=_get_auth_header(access_token),
+        )
+        return r
+
+    data = _request().json()
+    if "errors" in data:
+        raise ValueError(data)
+    data = data["data"]
+    # pop rate limits
+    rate_limit = data.pop("rateLimit", {"cost": 0, "remaining": 0})
+    return data, rate_limit
+
+
+def _get_graphql_pages(
+    access_token: str, query: str, variables: DictStrAny, node_type: str, max_items: int
+) -> Iterator[List[DictStrAny]]:
+    items_count = 0
+    while True:
+        data, rate_limit = _run_graphql_query(access_token, query, variables)
+        top_connection = _extract_top_connection(data, node_type)
+        data_items = (
+            top_connection["nodes"]
+            if "nodes" in top_connection
+            else top_connection["edges"]
+        )
+        items_count += len(data_items)
+        print(
+            f'Got {len(data_items)}/{items_count} {node_type}s, query cost {rate_limit["cost"]}, remaining credits: {rate_limit["remaining"]}'
+        )
+        if data_items:
+            yield data_items
+        else:
+            return
+        # print(data["repository"][node_type]["pageInfo"]["endCursor"])
+        variables["page_after"] = _extract_top_connection(data, node_type)["pageInfo"][
+            "endCursor"
+        ]
+        if max_items and items_count >= max_items:
+            print(f"Max items limit reached: {items_count} >= {max_items}")
+            return
+
+
+def _get_comment_reaction(comment_ids: List[str], access_token: str) -> StrAny:
+    """Builds a query from a list of comment nodes and returns associated reactions."""
+    idx = 0
+    data: DictStrAny = {}
+    for page_chunk in chunks(comment_ids, 50):
+        subs = []
+        for comment_id in page_chunk:
+            subs.append(COMMENT_REACTIONS_QUERY % (idx, comment_id))
+            idx += 1
+        subs.append(RATE_LIMIT)
+        query = "{" + ",\n".join(subs) + "}"
+        # print(query)
+        page, rate_limit = _run_graphql_query(access_token, query, {})
+        print(
+            f'Got {len(page)} comments, query cost {rate_limit["cost"]}, remaining credits: {rate_limit["remaining"]}'
+        )
+        data.update(page)
+    return data

ingestr/src/github/queries.py

@@ -0,0 +1,115 @@
+RATE_LIMIT = """
+  rateLimit {
+    limit
+    cost
+    remaining
+    resetAt
+  }
+"""
+
+ISSUES_QUERY = """
+query($owner: String!, $name: String!, $issues_per_page: Int!, $first_reactions: Int!, $first_comments: Int!, $page_after: String) {
+  repository(owner: $owner, name: $name) {
+    %s(first: $issues_per_page, orderBy: {field: CREATED_AT, direction: DESC}, after: $page_after) {
+      totalCount
+      pageInfo {
+        endCursor
+        startCursor
+      }
+      nodes {
+        # id
+        number
+        url
+        title
+        body
+        author {login avatarUrl url}
+        authorAssociation
+        closed
+        closedAt
+        createdAt
+        state
+        updatedAt
+        reactions(first: $first_reactions) {
+          totalCount
+          nodes {
+            # id
+            user {login avatarUrl url}
+            content
+            createdAt
+          }
+        }
+        comments(first: $first_comments) {
+          totalCount
+          nodes {
+            id
+            url
+            body
+            author {avatarUrl login url}
+            authorAssociation
+            createdAt
+            reactionGroups {content createdAt}
+            # reactions(first: 0) {
+            #   totalCount
+            #   nodes {
+            #     # id
+            #     user {login avatarUrl url}
+            #     content
+            #     createdAt
+            #   }
+            # }
+          }
+        }
+      }
+    }
+  }
+  rateLimit {
+    limit
+    cost
+    remaining
+    resetAt
+  }
+}
+"""
+
+COMMENT_REACTIONS_QUERY = """
+node_%s: node(id:"%s") {
+     ... on IssueComment {
+      id
+      reactions(first: 100) {
+        totalCount
+        nodes {
+            user {login avatarUrl url}
+            content
+            createdAt
+          }
+      }
+    }
+  }
+"""
+
+STARGAZERS_QUERY = """
+query($owner: String!, $name: String!, $items_per_page: Int!, $page_after: String) {
+  repository(owner: $owner, name: $name) {
+    stargazers(first: $items_per_page, orderBy: {field: STARRED_AT, direction: DESC}, after: $page_after) {
+      pageInfo {
+        endCursor
+        startCursor
+      }
+      edges {
+        starredAt
+        node {
+          login
+          avatarUrl
+          url
+        }
+      }
+    }
+  }
+  rateLimit {
+    limit
+    cost
+    remaining
+    resetAt
+  }
+}
+"""

ingestr/src/github/settings.py

@@ -0,0 +1,10 @@
+"""Github source settings and constants."""
+
+START_DATE = "1970-01-01T00:00:00Z"
+
+# rest queries
+REST_API_BASE_URL = "https://api.github.com"
+REPO_EVENTS_PATH = "/repos/%s/%s/events"
+
+# graphql queries
+GRAPHQL_API_BASE_URL = "https://api.github.com/graphql"

ingestr/src/google_analytics/__init__.py

@@ -0,0 +1,70 @@
+"""
+Defines all the sources and resources needed for Google Analytics V4
+"""
+
+from typing import List, Optional, Union
+
+import dlt
+from dlt.common.typing import DictStrAny
+from dlt.sources import DltResource
+from dlt.sources.credentials import GcpOAuthCredentials, GcpServiceAccountCredentials
+from google.analytics.data_v1beta import BetaAnalyticsDataClient
+
+from .helpers import basic_report
+
+
+@dlt.source(max_table_nesting=0)
+def google_analytics(
+    datetime: str,
+    credentials: Union[
+        GcpOAuthCredentials, GcpServiceAccountCredentials
+    ] = dlt.secrets.value,
+    property_id: int = dlt.config.value,
+    queries: List[DictStrAny] = dlt.config.value,
+    start_date: Optional[str] = "2015-08-14",
+    rows_per_page: int = 10000,
+) -> List[DltResource]:
+    try:
+        property_id = int(property_id)
+    except ValueError:
+        raise ValueError(
+            f"{property_id} is an invalid google property id. Please use a numeric id, and not your Measurement ID like G-7F1AE12JLR"
+        )
+    if property_id == 0:
+        raise ValueError(
+            "Google Analytics property id is 0. Did you forget to configure it?"
+        )
+    if not rows_per_page:
+        raise ValueError("Rows per page cannot be 0")
+    # generate access token for credentials if we are using OAuth2.0
+    if isinstance(credentials, GcpOAuthCredentials):
+        credentials.auth("https://www.googleapis.com/auth/analytics.readonly")
+
+    # Build the service object for Google Analytics api.
+    client = BetaAnalyticsDataClient(credentials=credentials.to_native_credentials())
+    if len(queries) > 1:
+        raise ValueError(
+            "Google Analytics supports a single query ingestion at a time, please give only one query"
+        )
+    query = queries[0]
+
+    # always add "date" to dimensions so we are able to track the last day of a report
+    dimensions = query["dimensions"]
+    resource_name = query["resource_name"]
+
+    res = dlt.resource(
+        basic_report, name="basic_report", merge_key=datetime, write_disposition="merge"
+    )(
+        client=client,
+        rows_per_page=rows_per_page,
+        property_id=property_id,
+        dimensions=dimensions,
+        metrics=query["metrics"],
+        resource_name=resource_name,
+        start_date=start_date,
+        last_date=dlt.sources.incremental(
+            datetime
+        ),  # pass empty primary key to avoid unique checks, a primary key defined by the resource will be used
+    )
+
+    return [res]