ingestr 0.12.7__tar.gz → 0.12.9__tar.gz

{ingestr-0.12.7 → ingestr-0.12.9}/.githooks/pre-commit-hook.sh

@@ -13,11 +13,11 @@ secret_detected() {
 
 # use gitleaks binary if available
 # else fallback to using docker for running gitleaks
-CMD="gitleaks dir -v"
+CMD="gitleaks protect --staged -v"
 
 if [[ ! `which gitleaks`  ]]; then
     which docker > /dev/null || (echo "gitleaks or docker is required for running secrets scan." && exit 1)
-    CMD="docker run -v $PWD:$WORK_DIR -w $WORK_DIR --rm ghcr.io/gitleaks/gitleaks:latest dir -v"
+    CMD="docker run -v $PWD:$WORK_DIR -w $WORK_DIR --rm ghcr.io/gitleaks/gitleaks:latest protect --staged -v"
 fi
 
 $CMD || secret_detected

{ingestr-0.12.7 → ingestr-0.12.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.7
+Version: 0.12.9
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -22,6 +22,7 @@ Requires-Dist: dlt==1.5.0
 Requires-Dist: duckdb-engine==0.13.5
 Requires-Dist: duckdb==1.1.3
 Requires-Dist: facebook-business==20.0.0
+Requires-Dist: gcsfs==2024.10.0
 Requires-Dist: google-analytics-data==0.18.16
 Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0

{ingestr-0.12.7 → ingestr-0.12.9}/docs/.vitepress/config.mjs

@@ -102,6 +102,7 @@ export default defineConfig({
                 text: "Facebook Ads",
                 link: "/supported-sources/facebook-ads.md",
               },
+              { text: "Google Cloud Storage (GCS)", link: "/supported-sources/gcs.md" },
               { text: "Google Analytics", link: "/supported-sources/google_analytics.md" },
               { text: "GitHub", link: "/supported-sources/github.md" },
               { text: "Google Sheets", link: "/supported-sources/gsheets.md" },

{ingestr-0.12.7 → ingestr-0.12.9}/docs/supported-sources/adjust.md

@@ -34,6 +34,7 @@ Adjust source allows ingesting data from various sources:
 
 - `campaigns`: Retrieves data for a campaign, showing the app's revenue and network costs over multiple days.
 - `creatives`: Retrieves data for a creative assets, detailing the app's revenue and network costs across multiple days.
+- `events`: Retrieves data for [events](https://dev.adjust.com/en/api/rs-api/events/) and event slugs.
 - `custom`: Retrieves custom data based on the dimensions and metrics specified.
 
 ### Custom reports: `custom:<dimensions>:<metrics>[:<filters>]`

{ingestr-0.12.7 → ingestr-0.12.9}/docs/supported-sources/appstore.md

@@ -17,7 +17,8 @@ URI Parameters:
 * `key_path`: path to API private key
 * `key_id`: ID of the generated key
 * `issuer_id`: Issuer ID of the generated key
-* `app_id`: application ID of your app. You can specify `app_id` multiple times with different ids to ingest data for multiple apps.
+* `app_id`: optional, application ID of your app. You can specify `app_id` multiple times with different ids to ingest data for multiple apps.
+  * You can also define the app_id in the table name. For example, `app-downloads-detailed:12345,67890` will ingest data for app with id `12345` and `67890`.
 
 ## Setting up Appstore Integration
 

ingestr-0.12.9/docs/supported-sources/gcs.md

@@ -0,0 +1,62 @@
+# Google Cloud Storage
+
+[Google Cloud Storage](https://cloud.google.com/storage?hl=en) is an online file storage web service for storing and accessing data on Google Cloud Platform infrastructure. The service combines the performance and scalability of Google's cloud with advanced security and sharing capabilities. It is an Infrastructure as a Service (IaaS), comparable to Amazon S3. 
+
+## URI format
+
+The URI format for Google Cloud Storage is as follows:
+
+```plaintext
+gs://<bucket_name>?credentials_path=/path/to/service-account.json>
+```
+
+URI parameters:
+
+- `bucket_name`: The name of the bucket
+- `credentials_path`: path to file containing your Google Cloud [Service Account](https://cloud.google.com/iam/docs/service-account-overview)
+
+## Setting up a GCS Integration
+
+To use Google Cloud Storage source in `ingestr`, you will need:
+* A Google Cloud Project.
+* A Service Account with atleast [roles/storage.objectUser](https://cloud.google.com/storage/docs/access-control/iam-roles) IAM permission.
+* A Service Account key file for the corresponding service account.
+
+For more information on how to create a Service Account or it's keys, see [Create service accounts](https://cloud.google.com/iam/docs/service-accounts-create) and [Create or delete service account keys](https://cloud.google.com/iam/docs/keys-create-delete) on Google Cloud docs.
+
+## Example
+
+Let's assume that:
+* Service account key in available in the current directory, under the filename `service_account.json`. 
+* The bucket you want to load data from is called `my-org-bucket`
+* The source file is available at `/data/latest/dump.csv`
+* The data needs to be saved in a DuckDB database called `local.db`
+* The destination table name will be `public.latest_dump`
+
+You can run the following command line to achieve this:
+
+```sh
+ingestr ingest \
+    --source-uri "gs://my-org-bucket?credentials_path=$PWD/service_account.json" \
+    --source-table "/data/latest/dump.csv" \
+    --dest-uri "duckdb:///local.db" \
+    --dest-table "public.latest_dump"
+```
+
+## Supported File Formats
+`gs` source only supports loading files in the following formats:
+* `csv`: Comma Separated Values (supports Tab Separated Values as well)
+* `parquet`: [Apache Parquet](https://parquet.apache.org/) storage format.
+* `jsonl`: Line delimited JSON. see [https://jsonlines.org/](https://jsonlines.org/)
+
+## File Pattern
+`ingestr` supports [glob](https://en.wikipedia.org/wiki/Glob_(programming)) like pattern matching for `gs` source.
+This allows for a powerful pattern matching mechanism that allows you to specify multiple files in a single `--source-table`.
+
+Below are some examples of path patterns, each path pattern is a reference from the root of the bucket:
+
+- `**/*.csv`: Retrieves all the CSV files, regardless of how deep they are within the folder structure.
+- `*.csv`: Retrieves all the CSV files from the first level of a folder.
+- `myFolder/**/*.jsonl`: Retrieves all the JSONL files from anywhere under `myFolder`.
+- `myFolder/mySubFolder/users.parquet`: Retrieves the `users.parquet` file from `mySubFolder`.
+- `employees.jsonl`: Retrieves the `employees.jsonl` file from the root level of the bucket.

{ingestr-0.12.7 → ingestr-0.12.9}/docs/supported-sources/s3.md

@@ -9,7 +9,7 @@ ingestr supports S3 as a source.
 The URI format for S3 is as follows:
 
 ```plaintext
-s3://<bucket_name>/<path_to_file>?access_key_id=<access_key_id>&secret_access_key=<secret_access_key>
+s3://<bucket_name>?access_key_id=<access_key_id>&secret_access_key=<secret_access_key>
 ```
 
 URI parameters:
@@ -25,7 +25,11 @@ S3 requires an `access_key_id` and a `secret_access_key` to access the bucket. P
 For example, if your `access_key_id` is `AKC3YOW7E`, `secret_access_key` is `XCtkpL5B`, bucket name is `my_bucket`, and `path_to_files` is `students/students_details.csv`, here's a sample command that will copy the data from the S3 bucket into a DuckDB database:
 
 ```sh
-ingestr ingest --source-uri 's3://my_bucket/students/students_details.csv?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' --source-table 'students_details' --dest-uri duckdb:///s3.duckdb --dest-table 'dest.students_details'
+ingestr ingest \
+    --source-uri 's3://my_bucket?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \
+    --source-table '/students/students_details.csv' \
+    --dest-uri duckdb:///s3.duckdb \
+    --dest-table 'dest.students_details'
 ```
 
 The result of this command will be a table in the DuckDB database in the path `s3.duckdb`.

{ingestr-0.12.7 → ingestr-0.12.9}/ingestr/src/adjust/__init__.py

@@ -56,6 +56,11 @@ def adjust_source(
             filters=filters,
         )
 
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def events():
+        adjust_api = AdjustAPI(api_key=api_key)
+        yield adjust_api.fetch_events()
+
     @dlt.resource(write_disposition="merge", merge_key="day")
     def creatives():
         adjust_api = AdjustAPI(api_key=api_key)
@@ -68,7 +73,7 @@ def adjust_source(
         )
 
     if not dimensions:
-        return campaigns, creatives
+        return campaigns, creatives, events
 
     merge_key = merge_key
     type_hints = {}
@@ -100,4 +105,4 @@ def adjust_source(
             filters=filters,
         )
 
-    return campaigns, creatives, custom
+    return campaigns, creatives, custom, events

{ingestr-0.12.7 → ingestr-0.12.9}/ingestr/src/adjust/adjust_helpers.py

@@ -28,10 +28,20 @@ DEFAULT_METRICS = [
 ]
 
 
+def retry_on_limit(response: requests.Response, exception: BaseException) -> bool:
+    return response.status_code == 429
+
+
 class AdjustAPI:
     def __init__(self, api_key):
         self.api_key = api_key
-        self.uri = "https://automate.adjust.com/reports-service/report"
+        self.request_client = Client(
+            request_timeout=8.0,
+            raise_for_status=False,
+            retry_condition=retry_on_limit,
+            request_max_attempts=12,
+            request_backoff_factor=2,
+        ).session
 
     def fetch_report_data(
         self,
@@ -62,20 +72,11 @@ class AdjustAPI:
                 f"Invalid date range: Start date ({start_date}) must be earlier than end date ({end_date})."
             )
 
-        def retry_on_limit(
-            response: requests.Response, exception: BaseException
-        ) -> bool:
-            return response.status_code == 429
-
-        request_client = Client(
-            request_timeout=8.0,
-            raise_for_status=False,
-            retry_condition=retry_on_limit,
-            request_max_attempts=12,
-            request_backoff_factor=2,
-        ).session
-
-        response = request_client.get(self.uri, headers=headers, params=params)
+        response = self.request_client.get(
+            "https://automate.adjust.com/reports-service/report",
+            headers=headers,
+            params=params,
+        )
         if response.status_code == 200:
             result = response.json()
             items = result.get("rows", [])
@@ -83,6 +84,17 @@ class AdjustAPI:
         else:
             raise HTTPError(f"Request failed with status code: {response.status_code}")
 
+    def fetch_events(self):
+        headers = {"Authorization": f"Bearer {self.api_key}"}
+        response = self.request_client.get(
+            "https://automate.adjust.com/reports-service/events", headers=headers
+        )
+        if response.status_code == 200:
+            result = response.json()
+            yield result
+        else:
+            raise HTTPError(f"Request failed with status code: {response.status_code}")
+
 
 def parse_filters(filters_raw: str) -> dict:
     # Parse filter string like "key1=value1,key2=value2,value3,value4"

{ingestr-0.12.7 → ingestr-0.12.9}/ingestr/src/appstore/__init__.py

@@ -28,6 +28,10 @@ def app_store(
     start_date: Optional[datetime] = None,
     end_date: Optional[datetime] = None,
 ) -> Iterable[DltResource]:
+    if start_date and start_date.tzinfo is not None:
+        start_date = start_date.replace(tzinfo=None)
+    if end_date and end_date.tzinfo is not None:
+        end_date = end_date.replace(tzinfo=None)
     for resource in RESOURCES:
         yield dlt.resource(
             get_analytics_reports,

{ingestr-0.12.7 → ingestr-0.12.9}/ingestr/src/factory.py

@@ -25,6 +25,7 @@ from ingestr.src.sources import (
     ChessSource,
     DynamoDBSource,
     FacebookAdsSource,
+    GCSSource,
     GitHubSource,
     GoogleAnalyticsSource,
     GoogleSheetsSource,
@@ -124,6 +125,7 @@ class SourceDestinationFactory:
         "tiktok": TikTokSource,
         "googleanalytics": GoogleAnalyticsSource,
         "appstore": AppleAppStoreSource,
+        "gs": GCSSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

{ingestr-0.12.7 → ingestr-0.12.9}/ingestr/src/filesystem/__init__.py

@@ -39,8 +39,6 @@ def readers(
     filesystem_resource = filesystem(bucket_url, credentials, file_glob=file_glob)
     filesystem_resource.apply_hints(
         incremental=dlt.sources.incremental("modification_date"),
-        range_end="closed",
-        range_start="closed",
     )
     return (
         filesystem_resource | dlt.transformer(name="read_csv")(_read_csv),

{ingestr-0.12.7 → ingestr-0.12.9}/ingestr/src/sources.py

@@ -17,6 +17,8 @@ from typing import (
 from urllib.parse import ParseResult, parse_qs, quote, urlparse
 
 import dlt
+import gcsfs  # type: ignore
+import s3fs # type: ignore
 import pendulum
 from dlt.common.configuration.specs import (
     AwsCredentials,
@@ -1091,19 +1093,17 @@ class S3Source:
         bucket_name = parsed_uri.hostname
         if not bucket_name:
             raise ValueError(
-                "Invalid S3 URI: The bucket name is missing. Ensure your S3 URI follows the format 's3://bucket-name/path/to/file"
+                "Invalid S3 URI: The bucket name is missing. Ensure your S3 URI follows the format 's3://bucket-name"
             )
         bucket_url = f"s3://{bucket_name}"
 
-        path_to_file = parsed_uri.path.lstrip("/")
+        path_to_file = parsed_uri.path.lstrip("/") or table.lstrip("/")
         if not path_to_file:
-            raise ValueError(
-                "Invalid S3 URI: The file path is missing. Ensure your S3 URI follows the format 's3://bucket-name/path/to/file"
-            )
+            raise ValueError("--source-table must be specified")
 
-        aws_credentials = AwsCredentials(
-            aws_access_key_id=access_key_id[0],
-            aws_secret_access_key=TSecretStrValue(secret_access_key[0]),
+        fs = s3fs.S3FileSystem(
+            key=access_key_id[0],
+            secret=secret_access_key[0],
         )
 
         file_extension = path_to_file.split(".")[-1]
@@ -1119,7 +1119,7 @@ class S3Source:
             )
 
         return readers(
-            bucket_url=bucket_url, credentials=aws_credentials, file_glob=path_to_file
+            bucket_url, fs, path_to_file
         ).with_resources(endpoint)
 
 
@@ -1503,3 +1503,69 @@ class AppleAppStoreSource:
             raise UnsupportedResourceError(table, "AppStore")
 
         return src.with_resources(table)
+
+
+class GCSSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "GCS takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        credentials_path = params.get("credentials_path")
+        credentials_base64 = params.get("credentials_base64")
+        credentials_available = any(
+            map(
+                lambda x: x is not None,
+                [credentials_path, credentials_base64],
+            )
+        )
+        if credentials_available is False:
+            raise MissingValueError("credentials_path or credentials_base64", "GCS")
+
+        bucket_name = parsed_uri.hostname
+        if not bucket_name:
+            raise ValueError(
+                "Invalid GCS URI: The bucket name is missing. Ensure your GCS URI follows the format 'gs://bucket-name/path/to/file"
+            )
+        bucket_url = f"gs://{bucket_name}/"
+
+        path_to_file = parsed_uri.path.lstrip("/") or table.lstrip("/")
+        if not path_to_file:
+            raise ValueError("--source-table must be specified")
+
+        credentials = None
+        if credentials_path:
+            credentials = credentials_path[0]
+        else:
+            credentials = json.loads(base64.b64decode(credentials_base64[0]).decode())  # type: ignore
+
+        # There's a compatiblity issue between google-auth, dlt and gcsfs
+        # that makes it difficult to use google.oauth2.service_account.Credentials
+        # (The RECOMMENDED way of passing service account credentials)
+        # directly with gcsfs. As a workaround, we construct the GCSFileSystem
+        # and pass it directly to filesystem.readers.
+        fs = gcsfs.GCSFileSystem(
+            token=credentials,
+        )
+
+        file_extension = path_to_file.split(".")[-1]
+        if file_extension == "csv":
+            endpoint = "read_csv"
+        elif file_extension == "jsonl":
+            endpoint = "read_jsonl"
+        elif file_extension == "parquet":
+            endpoint = "read_parquet"
+        else:
+            raise ValueError(
+                "GCS Source only supports specific formats files: csv, jsonl, parquet"
+            )
+
+        return readers(
+            bucket_url, fs, path_to_file
+        ).with_resources(endpoint)

ingestr-0.12.9/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.12.9"

{ingestr-0.12.7 → ingestr-0.12.9}/requirements.txt

@@ -40,4 +40,4 @@ pyathena==3.9.0
 google-analytics-data==0.18.16
 asana==3.2.3
 dataclasses-json==0.6.7
-
+gcsfs==2024.10.0

ingestr-0.12.7/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.12.7"