ingestr 0.12.3__tar.gz → 0.12.4__tar.gz

{ingestr-0.12.3 → ingestr-0.12.4}/.githooks/pre-commit-hook.sh

@@ -17,7 +17,7 @@ CMD="gitleaks dir -v"
 
 if [[ ! `which gitleaks`  ]]; then
     which docker > /dev/null || (echo "gitleaks or docker is required for running secrets scan." && exit 1)
-    CMD="docker run -v $PWD:$WORK_DIR -w $WORK_DIR ghcr.io/gitleaks/gitleaks:latest dir -v"
+    CMD="docker run -v $PWD:$WORK_DIR -w $WORK_DIR --rm ghcr.io/gitleaks/gitleaks:latest dir -v"
 fi
 
 $CMD || secret_detected

{ingestr-0.12.3 → ingestr-0.12.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.3
+Version: 0.12.4
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -21,6 +21,7 @@ Requires-Dist: dlt==1.4.0
 Requires-Dist: duckdb-engine==0.13.5
 Requires-Dist: duckdb==1.1.3
 Requires-Dist: facebook-business==20.0.0
+Requires-Dist: google-analytics-data==0.18.15
 Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: mysql-connector-python==9.1.0

{ingestr-0.12.3 → ingestr-0.12.4}/docs/.vitepress/config.mjs

@@ -101,6 +101,8 @@ export default defineConfig({
                 text: "Facebook Ads",
                 link: "/supported-sources/facebook-ads.md",
               },
+              { text: "Google Analytics", link: "/supported-sources/google_analytics.md" },
+              { text: "GitHub", link: "/supported-sources/github.md" },
               { text: "Google Sheets", link: "/supported-sources/gsheets.md" },
               { text: "Gorgias", link: "/supported-sources/gorgias.md" },
               { text: "HubSpot", link: "/supported-sources/hubspot.md" },

ingestr-0.12.4/docs/commands/ingest.md

@@ -0,0 +1,110 @@
+# `ingestr ingest`
+
+The `ingest` command is a core feature of the `ingestr` tool, allowing users to transfer data from a source to a destination with optional support for incremental updates.
+
+## Example
+
+The following example demonstrates how to use the `ingest` command to transfer data from a source to a destination.
+
+```bash
+ingestr ingest \
+   --source-uri '<your-source-uri-here>' \
+   --source-table '<your-schema>.<your-table>' \
+   --dest-uri '<your-destination-uri-here>'
+```
+
+## Required flags
+
+- `--source-uri TEXT`: Required. Specifies the URI of the data source.
+- `--dest-uri TEXT`: Required. Specifies the URI of the destination where data will be ingested.
+- `--source-table TEXT`: Required. Defines the source table to fetch data from.
+
+## Optional flags
+
+- `--dest-table TEXT`: Designates the destination table to save the data. If not specified, defaults to the value of `--source-table`.
+- `--incremental-key TEXT`: Identifies the key used for incremental data strategies. Defaults to `None`.
+- `--incremental-strategy TEXT`: Defines the strategy for incremental updates. Options include `replace`, `append`, `delete+insert`, or `merge`. The default strategy is `replace`.
+- `--interval-start`: Sets the start of the interval for the incremental key. Defaults to `None`.
+- `--interval-end`: Sets the end of the interval for the incremental key. Defaults to `None`.
+- `--primary-key TEXT`: Specifies the primary key for the merge operation. Defaults to `None`.
+- `--columns <column_name>:<column_type>`: Specifies the columns to be ingested. Defaults to `None`.
+
+The `interval-start` and `interval-end` options support various datetime formats, here are some examples:
+- `%Y-%m-%d`: `2023-01-31`
+- `%Y-%m-%dT%H:%M:%S`: `2023-01-31T15:00:00`
+- `%Y-%m-%dT%H:%M:%S%z`: `2023-01-31T15:00:00+00:00`
+- `%Y-%m-%dT%H:%M:%S.%f`: `2023-01-31T15:00:00.000123`
+- `%Y-%m-%dT%H:%M:%S.%f%z`: `2023-01-31T15:00:00.000123+00:00`
+
+> [!INFO]
+> For the details around the incremental key and the various strategies, please refer to the [Incremental Loading](../getting-started/incremental-loading.md) section.
+
+## General flags
+
+- `--help`: Displays the help message and exits the command.
+
+## Examples
+
+### Ingesting a CSV file to DuckDB
+
+```bash
+ingestr ingest \
+   --source-uri 'csv://input.csv' \
+   --source-table 'sample' \
+   --dest-uri 'duckdb://output.duckdb'
+```
+
+### Copy a table from Postgres to DuckDB
+
+```bash
+ingestr ingest \
+   --source-uri 'postgresql://myuser:mypassword@localhost:5432/mydatabase?sslmode=disable' \
+   --source-table 'public.input_table' \
+   --dest-uri 'duckdb://output.duckdb' \
+   --dest-table 'public.output_table'
+```
+
+### Incrementally ingest a table from Postgres to BigQuery
+
+```bash
+ingestr ingest 
+   --source-uri 'postgresql://myuser:mypassword@localhost:5432/mydatabase?sslmode=disable' \
+   --source-table 'public.users' \
+   --dest-uri 'bigquery://my_project?credentials_path=/path/to/service/account.json&location=EU' \
+   --dest-table 'raw.users' \
+   --incremental-key 'updated_at' \
+   --incremental-strategy 'delete+insert'
+```
+
+### Load an interval of data from Postgres to BigQuery using a date column
+
+```bash
+ingestr ingest 
+   --source-uri 'postgresql://myuser:mypassword@localhost:5432/mydatabase?sslmode=disable' \
+   --source-table 'public.users' \
+   --dest-uri 'bigquery://my_project?credentials_path=/path/to/service/account.json&location=EU' \
+   --dest-table 'raw.users' \
+   --incremental-key 'dt' \
+   --incremental-strategy 'delete+insert' \
+   --interval-start '2023-01-01' \
+   --interval-end '2023-01-31' \
+   --columns 'dt:date'
+```
+
+### Load a specific query from Postgres to Snowflake
+
+```bash
+ingestr ingest 
+   --source-uri 'postgresql://myuser:mypassword@localhost:5432/mydatabase?sslmode=disable' \
+   --dest-uri 'snowflake://user:password@account/dbname?warehouse=COMPUTE_WH&role=my_role' \
+   --source-table 'query:SELECT * FROM public.users as pu JOIN public.orders as o ON pu.id = o.user_id WHERE pu.dt BETWEEN :interval_start AND :interval_end' \
+   --dest-table 'raw.users' \
+   --incremental-key 'dt' \
+   --incremental-strategy 'delete+insert' \
+   --interval-start '2023-01-01' \
+   --interval-end '2023-01-31' \
+   --columns 'dt:date'
+```
+
+> [!INFO]
+> For more examples, please refer to the specific platforms' documentation on the sidebar.

ingestr-0.12.4/docs/supported-sources/github.md

@@ -0,0 +1,49 @@
+# GitHub
+
+[GitHub](https://github.com/) is a developer platform that allows developers to create, store, manage and share their code.
+
+ingestr supports GitHub as a source.
+
+## URI format
+
+The URI format for GitHub is as follows:
+
+```plaintext
+github://?access_token=<access_token>&owner=<owner>&repo=<repo>
+```
+
+URI parameters:
+
+- `access_token`: Access Token used for authentication with the GitHub API
+- `owner`: Refers to the owner of the repository
+- `repo`: Refers to the name of the repository
+
+
+## Setting up a GitHub Integration
+
+GitHub requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/github#setup-guide).
+
+Once you complete the guide, you should have an access token. Let's say your access token is `ghp_test_1234`, the owner is `max`, and the name of the repository is `test_example`. Here is a sample command that will copy the data from GitHub into a DuckDB database:
+
+```sh
+ingestr ingest --source-uri 'github://?access_token=ghp_test_1234&owner=max&repo=test_example' --source-table 'issues' --dest-uri duckdb:///github.duckdb --dest-table 'dest.issues'
+```
+
+This is a sample command that will copy the data from the GitHub source to DuckDB.
+
+<img alt="github_img" src="../media/github.png" />
+
+## Tables
+
+GitHub source allows ingesting the following sources into separate tables:
+
+- `issues`: Retrieves data about issues, their associated comments, and subsequent reactions
+- `pull_requests`: Retrieves all pull requests
+- `repo_events`: Retrieves all the repo events associated with the repository
+- `stargazers`: Retrieves all stargazers
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+> [!WARNING]
+> GitHub does not support incremental loading for many endpoints in its APIs, which means ingestr will load endpoints incrementally if they support it, and do a full-refresh if not.
+ 

ingestr-0.12.4/docs/supported-sources/google_analytics.md

@@ -0,0 +1,44 @@
+# Google Analytics
+[Google Analytics](https://marketingplatform.google.com/about/analytics/) is a service for web analytics that tracks and provides data regarding user engagement with your website or application.
+
+ingestr supports Google Analytics as a source.
+
+## URI format
+The URI format for Google Analytics is as follows:
+
+```plaintext
+googleanalytics://?credentials_path=/path/to/service/account.json&property_id=<property_id>
+```
+
+URI parameters:
+- `credentials_path`: The path to the service account JSON file.
+- `property_id`: It is a unique number that identifies a particular property on Google Analytics. [Follow this guide](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id) if you don't know your property ID.
+
+## Setting up an Google Analytics Integration
+Google Analytics requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/google_analytics#grab-google-service-account-credentials). Once you complete the guide, you should have an `.json` file  and `project_id`. 
+
+## Table: Custom Reports
+Custom reports allow you to retrieve data based on specific `dimensions` and  `metrics`.
+
+Custom Table Format:
+```
+custom:<dimensions>:<metrics>
+```
+
+### Parameters:
+- `dimensions`(required): A comma-separated list of [dimensions](https://developers.google.com/analytics/devguides/reporting/data/v1/exploration-api-schema#dimensions) to retrieve.
+- `metrics`(required): A comma-separated list of [metrics](https://developers.google.com/analytics/devguides/reporting/data/v1/exploration-api-schema#metrics) to retrieve.
+
+### Example
+
+```sh
+ingestr ingest \
+    --source-uri "googleanalytics://?credentials_path="ingestr/src/g_analytics.json&property_id=id123" \
+    --source-table "custom:city,date:clicks,activeUsers,newUsers" \
+    --dest-uri "duckdb:///analytics.duckdb" \
+    --dest-table "dest.custom"
+```
+
+This command will retrieve report and save it to the `dest.custom` table in the DuckDB database.
+
+<img alt="google_analytics_img" src="../media/googleanalytics.png" />

{ingestr-0.12.3 → ingestr-0.12.4}/ingestr/main.py

@@ -32,7 +32,7 @@ DATE_FORMATS = [
 
 # https://dlthub.com/docs/dlt-ecosystem/file-formats/parquet#supported-destinations
 PARQUET_SUPPORTED_DESTINATIONS = [
-    "bigquery",
+    "athena" "bigquery",
     "duckdb",
     "snowflake",
     "databricks",
@@ -287,8 +287,14 @@ def ingest(
             envvar="SQL_EXCLUDE_COLUMNS",
         ),
     ] = [],  # type: ignore
+    columns: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            help="The column types to be used for the destination table in the format of 'column_name:column_type'",
+            envvar="COLUMNS",
+        ),
+    ] = None,  # type: ignore
 ):
-    # TODO(turtledev): can't we move this to the top of this file?
     import hashlib
     import tempfile
     from datetime import datetime
@@ -296,6 +302,7 @@ def ingest(
     import dlt
     import humanize
     import typer
+    from dlt.common.data_types import TDataType
     from dlt.common.destination import Destination
     from dlt.common.pipeline import LoadInfo
     from dlt.common.runtime.collector import Collector, LogCollector
@@ -345,7 +352,7 @@ def ingest(
             not in dlt_dest.capabilities().supported_loader_file_formats
         ):
             print(
-                f"[red]Loader file format {loader_file_format.value} is not supported by the destination.[/red]"
+                f"[red]Loader file format {loader_file_format.value} is not supported by the destination, available formats: {dlt_dest.capabilities().supported_loader_file_formats}.[/red]"
             )
             raise typer.Abort()
 
@@ -357,6 +364,23 @@ def ingest(
         else:
             executable(source)
 
+    def parse_columns(columns: list[str]) -> dict[str, TDataType]:
+        from typing import cast, get_args
+
+        possible_types = get_args(TDataType)
+
+        types: dict[str, TDataType] = {}
+        for column in columns:
+            for candidate in column.split(","):
+                column_name, column_type = candidate.split(":")
+                if column_type not in possible_types:
+                    print(
+                        f"[red]Column type '{column_type}' is not supported, supported types: {possible_types}.[/red]"
+                    )
+                    raise typer.Abort()
+                types[column_name] = cast(TDataType, column_type)
+        return types
+
     track(
         "command_triggered",
         {
@@ -399,12 +423,20 @@ def ingest(
         column_hints: dict[str, TColumnSchema] = {}
         original_incremental_strategy = incremental_strategy
 
+        if columns:
+            column_types = parse_columns(columns)
+            for column_name, column_type in column_types.items():
+                column_hints[column_name] = {"data_type": column_type}
+
         merge_key = None
         if incremental_strategy == IncrementalStrategy.delete_insert:
             merge_key = incremental_key
             incremental_strategy = IncrementalStrategy.merge
             if incremental_key:
-                column_hints[incremental_key] = {"merge_key": True}
+                if incremental_key not in column_hints:
+                    column_hints[incremental_key] = {}
+
+                column_hints[incremental_key]["merge_key"] = True
 
         m = hashlib.sha256()
         m.update(dest_table.encode("utf-8"))
@@ -491,6 +523,21 @@ def ingest(
         if factory.source_scheme == "sqlite":
             source_table = "main." + source_table.split(".")[-1]
 
+        if (
+            incremental_key
+            and incremental_key in column_hints
+            and "data_type" in column_hints[incremental_key]
+            and column_hints[incremental_key]["data_type"] == "date"
+        ):
+            # By default, ingestr treats the start and end dates as datetime objects. While this worked fine for many cases, if the
+            # incremental field is a date, the start and end dates cannot be compared to the incremental field, and the ingestion would fail.
+            # In order to eliminate this, we have introduced a new option to ingestr, --columns, which allows the user to specify the column types for the destination table.
+            # This way, ingestr will know the data type of the incremental field, and will be able to convert the start and end dates to the correct data type before running the ingestion.
+            if interval_start:
+                interval_start = interval_start.date()  # type: ignore
+            if interval_end:
+                interval_end = interval_end.date()  # type: ignore
+
         dlt_source = source.dlt_source(
             uri=source_uri,
             table=source_table,

{ingestr-0.12.3 → ingestr-0.12.4}/ingestr/src/factory.py

@@ -24,6 +24,8 @@ from ingestr.src.sources import (
     ChessSource,
     DynamoDBSource,
     FacebookAdsSource,
+    GitHubSource,
+    GoogleAnalyticsSource,
     GoogleSheetsSource,
     GorgiasSource,
     HubspotSource,
@@ -102,6 +104,7 @@ class SourceDestinationFactory:
         "gsheets": GoogleSheetsSource,
         "shopify": ShopifySource,
         "gorgias": GorgiasSource,
+        "github": GitHubSource,
         "chess": ChessSource,
         "stripe": StripeAnalyticsSource,
         "facebookads": FacebookAdsSource,
@@ -118,6 +121,7 @@ class SourceDestinationFactory:
         "dynamodb": DynamoDBSource,
         "asana": AsanaSource,
         "tiktok": TikTokSource,
+        "googleanalytics": GoogleAnalyticsSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr-0.12.4/ingestr/src/github/__init__.py

@@ -0,0 +1,149 @@
+"""Source that load github issues, pull requests and reactions for a specific repository via customizable graphql query. Loads events incrementally."""
+
+import urllib.parse
+from typing import Iterator, Optional, Sequence
+
+import dlt
+from dlt.common.typing import TDataItems
+from dlt.sources import DltResource
+
+from .helpers import get_reactions_data, get_rest_pages, get_stargazers
+
+
+@dlt.source
+def github_reactions(
+    owner: str,
+    name: str,
+    access_token: str = dlt.secrets.value,
+    items_per_page: int = 100,
+    max_items: Optional[int] = None,
+) -> Sequence[DltResource]:
+    """Get reactions associated with issues, pull requests and comments in the repo `name` with owner `owner`.
+
+    This source uses graphql to retrieve all issues (`issues` resource) and pull requests (`pull requests` resource) with the associated reactions (up to 100),
+    comments (up to 100) and reactions to comments (also up to 100). Internally graphql is used to retrieve data. It is cost optimized and you are able to retrieve the
+    data for fairly large repos quickly and cheaply.
+    You can and should change the queries in `queries.py` to include for example additional fields or connections. The source can be hacked to add more resources for other
+    repository nodes easily.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic access token. Will be injected from secrets if not provided.
+        items_per_page (int, optional): How many issues/pull requests to get in single page. Defaults to 100.
+        max_items (int, optional): How many issues/pull requests to get in total. None means All.
+        max_item_age_seconds (float, optional): Do not get items older than this. Defaults to None. NOT IMPLEMENTED
+
+    Returns:
+        Sequence[DltResource]: Two DltResources: `issues` with issues and `pull_requests` with pull requests
+    """
+    return (
+        dlt.resource(
+            get_reactions_data(
+                "issues",
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="issues",
+            write_disposition="replace",
+        ),
+        dlt.resource(
+            get_reactions_data(
+                "pullRequests",
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="pull_requests",
+            write_disposition="replace",
+        ),
+    )
+
+
+@dlt.source(max_table_nesting=0)
+def github_repo_events(
+    owner: str, name: str, access_token: Optional[str] = None
+) -> DltResource:
+    """Gets events for repository `name` with owner `owner` incrementally.
+
+    This source contains a single resource `repo_events` that gets given repository's events and dispatches them to separate tables with names based on event type.
+    The data is loaded incrementally. Subsequent runs will get only new events and append them to tables.
+    Please note that Github allows only for 300 events to be retrieved for public repositories. You should get the events frequently for the active repos.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic or fine-grained access token. If not provided, calls are made anonymously
+
+    Returns:
+        DltSource: source with the `repo_events` resource
+
+    """
+
+    # use naming function in table name to generate separate tables for each event
+    @dlt.resource(primary_key="id", table_name=lambda i: i["type"])
+    def repo_events(
+        last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "created_at", initial_value="1970-01-01T00:00:00Z", last_value_func=max
+        ),
+    ) -> Iterator[TDataItems]:
+        repos_path = (
+            f"/repos/{urllib.parse.quote(owner)}/{urllib.parse.quote(name)}/events"
+        )
+
+        for page in get_rest_pages(access_token, repos_path + "?per_page=100"):
+            yield page
+
+            # stop requesting pages if the last element was already older than initial value
+            # note: incremental will skip those items anyway, we just do not want to use the api limits
+            if last_created_at.start_out_of_range:
+                print(
+                    f"Overlap with previous run created at {last_created_at.initial_value}"
+                )
+                break
+
+    return repo_events
+
+
+@dlt.source
+def github_stargazers(
+    owner: str,
+    name: str,
+    access_token: str = dlt.secrets.value,
+    items_per_page: int = 100,
+    max_items: Optional[int] = None,
+) -> Sequence[DltResource]:
+    """Get stargazers in the repo `name` with owner `owner`.
+
+    This source uses graphql to retrieve all stargazers with the associated starred date,
+    Internally graphql is used to retrieve data. It is cost optimized and you are able to retrieve the
+    data for fairly large repos quickly and cheaply.
+
+    Args:
+        owner (str): The repository owner
+        name (str): The repository name
+        access_token (str): The classic access token. Will be injected from secrets if not provided.
+        items_per_page (int, optional): How many issues/pull requests to get in single page. Defaults to 100.
+        max_items (int, optional): How many issues/pull requests to get in total. None means All.
+
+    Returns:
+        Sequence[DltResource]: One DltResource: `stargazers`
+    """
+    return (
+        dlt.resource(
+            get_stargazers(
+                owner,
+                name,
+                access_token,
+                items_per_page,
+                max_items,
+            ),
+            name="stargazers",
+            write_disposition="replace",
+        ),
+    )