ingestr 0.13.54__tar.gz → 0.13.56__tar.gz

{ingestr-0.13.54 → ingestr-0.13.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.54
+Version: 0.13.56
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.54 → ingestr-0.13.56}/docs/.vitepress/config.mjs

@@ -123,6 +123,7 @@ export default defineConfig({
               },
               { text: "Frankfurter", link: "/supported-sources/frankfurter.md" },
               { text: "Freshdesk", link: "/supported-sources/freshdesk.md" },
+              { text: "Trustpilot", link: "/supported-sources/trustpilot.md" },
               { text: "Google Cloud Storage (GCS)", link: "/supported-sources/gcs.md" },
               { text: "Google Analytics", link: "/supported-sources/google_analytics.md" },
               { text: "Google Ads", link: "/supported-sources/google-ads.md" },

ingestr-0.13.56/docs/supported-sources/facebook-ads.md

@@ -0,0 +1,202 @@
+# Facebook Ads
+
+Facebook Ads is the advertising platform that helps users to create targeted ads on Facebook, Instagram and Messenger.
+
+ingestr supports Facebook Ads as a source.
+
+## URI format
+
+The URI format for Facebook Ads is as follows:
+
+```plaintext
+facebookads://?access_token=<access_token>&account_id=<account_id>
+```
+
+URI parameters:
+
+- `access_token` is associated with Business Facebook App.
+- `account_id` is associated with Ad manager.
+
+Both are used for authentication with Facebook Ads API.
+
+The URI is used to connect to Facebook Ads API for extracting data.
+
+## Setting up a Facebook Ads Integration
+
+Facebook Ads requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/facebook_ads#setup-guide).
+
+Once you complete the guide, you should have an access token and an Account ID. Let's say your `access_token` is `abcdef` and `account_id` is `1234`, here's a sample command that will copy the data from Facebook Ads into a DuckDB database:
+
+```sh
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'campaigns' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.campaigns'
+```
+
+The result of this command will be a table in the `facebook.duckdb` database.
+
+## Tables
+
+Facebook Ads source allows ingesting the following sources into separate tables:
+
+- `campaigns`: Retrieves campaign data with fields:
+  - `id`
+  - `updated_time`
+  - `created_time`
+  - `name`
+  - `status`
+  - `effective_status`
+  - `objective`
+  - `start_time`
+  - `stop_time`
+  - `daily_budget`
+  - `lifetime_budget`
+
+- `ad_sets`: Retrieves ad set data with fields:
+  - `id`
+  - `updated_time`
+  - `created_time`
+  - `name`
+  - `status`
+  - `effective_status`
+  - `campaign_id`
+  - `start_time`
+  - `end_time`
+  - `daily_budget`
+  - `lifetime_budget`
+  - `optimization_goal`
+  - `promoted_object`
+  - `billing_event`
+  - `bid_amount`
+  - `bid_strategy`
+  - `targeting`
+
+- `leads`: Retrieves lead data with fields:
+  - `id`
+  - `created_time`
+  - `ad_id`
+  - `ad_name`
+  - `adset_id`
+  - `adset_name`
+  - `campaign_id`
+  - `campaign_name`
+  - `form_id`
+  - `field_data`
+
+- `ads_creatives`: Retrieves ad creative data with fields:
+  - `id`
+  - `name`
+  - `status`
+  - `thumbnail_url`
+  - `object_story_spec`
+  - `effective_object_story_id`
+  - `call_to_action_type`
+  - `object_type`
+  - `template_url`
+  - `url_tags`
+  - `instagram_actor_id`
+  - `product_set_id`
+
+- `ads`: Retrieves ad data with fields:
+  - `id`
+  - `updated_time`
+  - `created_time`
+  - `name`
+  - `status`
+  - `effective_status`
+  - `adset_id`
+  - `campaign_id`
+  - `creative`
+  - `targeting`
+  - `tracking_specs`
+  - `conversion_specs`
+
+- `facebook_insights`: Retrieves insights data with fields:
+  - `campaign_id`
+  - `adset_id`
+  - `ad_id`
+  - `date_start`
+  - `date_stop`
+  - `reach`
+  - `impressions`
+  - `frequency`
+  - `clicks`
+  - `unique_clicks`
+  - `ctr`
+  - `unique_ctr`
+  - `cpc`
+  - `cpm`
+  - `cpp`
+  - `spend`
+  - `actions`
+  - `action_values`
+  - `cost_per_action_type`
+  - `website_ctr`
+  - `account_currency`
+  - `ad_click_actions`
+  - `ad_name`
+  - `adset_name`
+  - `campaign_name`
+  - `country`
+  - `dma`
+  - `full_view_impressions`
+  - `full_view_reach`
+  - `inline_link_click_ctr`
+  - `outbound_clicks`
+  - `social_spend`
+  - `conversions`
+  - `video_thruplay_watched_actions`
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+### Facebook Insights Custom Configuration
+
+The `facebook_insights` table supports advanced configuration for breakdowns and custom metrics:
+
+#### Format Options
+
+1. **Default usage**: `facebook_insights`
+   - Uses default breakdown and default fields
+
+2. **Custom breakdown**: `facebook_insights:breakdown_type`
+   - Uses specified breakdown with default fields
+
+3. **Custom breakdown + metrics**: `facebook_insights:breakdown_type:metric1,metric2,metric3`
+   - Uses specified breakdown with custom metrics
+
+#### Available Breakdown Types
+
+- `ads_insights` (default)
+- `ads_insights_age_and_gender`
+- `ads_insights_country`
+- `ads_insights_platform_and_device`
+- `ads_insights_region`
+- `ads_insights_dma`
+- `ads_insights_hourly_advertiser`
+
+#### Examples
+
+```sh
+# Default facebook_insights
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.insights'
+
+# Age and gender breakdown with default metrics
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:ads_insights_age_and_gender' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.insights_demographics'
+
+# Country breakdown with custom metrics
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:ads_insights_country:impressions,clicks,spend,reach,cpm,ctr' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.insights_by_country'
+```

{ingestr-0.13.54 → ingestr-0.13.56}/docs/supported-sources/s3.md

@@ -134,3 +134,49 @@ The `<file-glob-pattern>` in the `--source-table` argument allows for flexible f
 | `bucket/myFolder/**/*.jsonl`                | Retrieves all JSONL files recursively from the `myFolder` directory and its subdirectories in `s3://bucket`. |
 | `bucket/myFolder/mySubFolder/users.parquet` | Retrieves the specific `users.parquet` file from the `myFolder/mySubFolder/` path in `s3://bucket`.        |
 | `bucket/employees.jsonl`                    | Retrieves the `employees.jsonl` file located at the root level of the `s3://bucket`.                       |
+
+
+### Working with compressed files
+
+`ingestr` automatically detects and handles gzipped files in your S3 bucket. You can load data from compressed files with the `.gz` extension without any additional configuration.
+
+For example, to load data from a gzipped CSV file:
+
+```sh
+ingestr ingest \
+    --source-uri 's3://?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \
+    --source-table 'my_bucket/logs/event-data.csv.gz' \
+    --dest-uri duckdb:///compressed_data.duckdb \
+    --dest-table 'logs.events'
+```
+
+You can also use glob patterns to load multiple compressed files:
+
+```sh
+ingestr ingest \
+    --source-uri 's3://?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \
+    --source-table 'my_bucket/logs/**/*.csv.gz' \
+    --dest-uri duckdb:///compressed_data.duckdb \
+    --dest-table 'logs.events'
+```
+
+### File type hinting
+
+If your files are properly encoded but lack the correct file extension (CSV, JSONL, or Parquet), you can provide a file type hint to inform `ingestr` about the format of the files. This is done by appending a fragment identifier (`#format`) to the end of the path in your `--source-table` parameter.
+
+For example, if you have JSONL-formatted log files stored in S3 with a non-standard extension:
+
+```
+--source-table "my_bucket/logs/event-data#jsonl"
+```
+
+This tells `ingestr` to process the files as JSONL, regardless of their actual extension.
+
+Supported format hints include:
+- `#csv` - For comma-separated values files
+- `#jsonl` - For line-delimited JSON files
+- `#parquet` - For Parquet format files
+
+::: tip
+File type hinting works with `gzip` compressed files as well.
+:::

ingestr-0.13.56/docs/supported-sources/trustpilot.md

@@ -0,0 +1,35 @@
+# Trustpilot
+
+[Trustpilot](https://www.trustpilot.com/) provides a platform for collecting and
+sharing customer reviews.
+
+ingestr supports Trustpilot as a source.
+
+## URI format
+
+The URI format for Trustpilot is:
+
+```
+trustpilot://<business_unit_id>?api_key=<api_key>
+```
+
+URI parameters:
+- `api_key`: Your Trustpilot API key.
+- `business_unit_id`: Identifier of the business unit whose reviews you want to fetch.
+
+## Example usage
+
+Assuming your `business_unit_id` is `123` and your API key is `key_abc`, you can ingest reviews into DuckDB using:
+
+```bash
+ingestr ingest --source-uri 'trustpilot://123?api_key=key_abc' --source-table 'reviews' --dest-uri duckdb:///trustpilot.duckdb --dest-table 'dest.reviews'
+```
+
+## Tables
+
+Currently the Trustpilot source exposes the following table:
+
+| Name    | Description                                 |
+| ------- | ------------------------------------------- |
+| reviews | Customer reviews for the specified business |
+

{ingestr-0.13.54 → ingestr-0.13.56}/ingestr/src/blob.py

@@ -6,6 +6,10 @@ BucketName: TypeAlias = str
 FileGlob: TypeAlias = str
 
 
+class UnsupportedEndpointError(Exception):
+    pass
+
+
 def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
     """
     parse the URI of a blob storage and
@@ -50,3 +54,23 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
         return "", parts[0]
 
     return parts[0], parts[1]
+
+
+def parse_endpoint(path: str) -> str:
+    """
+    Parse the endpoint kind from the URI.
+
+    kind is a file format. one of [csv, jsonl, parquet]
+    """
+    file_extension = path.split(".")[-1]
+    if file_extension == "gz":
+        file_extension = path.split(".")[-2]
+    if file_extension == "csv":
+        endpoint = "read_csv"
+    elif file_extension == "jsonl":
+        endpoint = "read_jsonl"
+    elif file_extension == "parquet":
+        endpoint = "read_parquet"
+    else:
+        raise UnsupportedEndpointError(f"Unsupported file format: {file_extension}")
+    return endpoint

ingestr-0.13.56/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.56"

{ingestr-0.13.54 → ingestr-0.13.56}/ingestr/src/facebook_ads/__init__.py

@@ -116,6 +116,8 @@ def facebook_insights_source(
     batch_size: int = 50,
     request_timeout: int = 300,
     app_api_version: str = None,
+    start_date: pendulum.DateTime | None = None,
+    end_date: pendulum.DateTime | None = None,
 ) -> DltResource:
     """Incrementally loads insight reports with defined granularity level, fields, breakdowns etc.
 
@@ -148,27 +150,32 @@ def facebook_insights_source(
         account_id, access_token, request_timeout, app_api_version
     )
 
-    # we load with a defined lag
-    initial_load_start_date = pendulum.today().subtract(days=initial_load_past_days)
-    initial_load_start_date_str = initial_load_start_date.isoformat()
+    if start_date is None:
+        start_date = pendulum.today().subtract(days=initial_load_past_days)
+
+    columns = {}
+    for field in fields:
+        if field in INSIGHT_FIELDS_TYPES:
+            columns[field] = INSIGHT_FIELDS_TYPES[field]
 
     @dlt.resource(
         primary_key=INSIGHTS_PRIMARY_KEY,
         write_disposition="merge",
-        columns=INSIGHT_FIELDS_TYPES,
+        columns=columns,
     )
     def facebook_insights(
         date_start: dlt.sources.incremental[str] = dlt.sources.incremental(
             "date_start",
-            initial_value=initial_load_start_date_str,
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
             range_end="closed",
             range_start="closed",
+            lag=attribution_window_days_lag * 24 * 60 * 60,  # Convert days to seconds
         ),
     ) -> Iterator[TDataItems]:
-        start_date = get_start_date(date_start, attribution_window_days_lag)
+        start_date = get_start_date(date_start)
         end_date = pendulum.now()
 
-        # fetch insights in incremental day steps
         while start_date <= end_date:
             query = {
                 "level": level,
@@ -193,7 +200,10 @@ def facebook_insights_source(
                     }
                 ],
             }
-            job = execute_job(account.get_insights(params=query, is_async=True))
+            job = execute_job(
+                account.get_insights(params=query, is_async=True),
+                insights_max_async_sleep_seconds=10,
+            )
             yield list(map(process_report_item, job.get_result()))
             start_date = start_date.add(days=time_increment_days)
 

{ingestr-0.13.54 → ingestr-0.13.56}/ingestr/src/facebook_ads/helpers.py

@@ -31,14 +31,13 @@ from .settings import (
 
 def get_start_date(
     incremental_start_date: dlt.sources.incremental[str],
-    attribution_window_days_lag: int = 7,
 ) -> pendulum.DateTime:
     """
     Get the start date for incremental loading of Facebook Insights data.
     """
     start_date: pendulum.DateTime = ensure_pendulum_datetime(
         incremental_start_date.start_value
-    ).subtract(days=attribution_window_days_lag)
+    )
 
     # facebook forgets insights so trim the lag and warn
     min_start_date = pendulum.today().subtract(
@@ -65,7 +64,6 @@ def process_report_item(item: AbstractObject) -> DictStrAny:
     for pki in INSIGHTS_PRIMARY_KEY:
         if pki not in d:
             d[pki] = "no_" + pki
-
     return d
 
 
@@ -138,7 +136,7 @@ def execute_job(
 ) -> AbstractCrudObject:
     status: str = None
     time_start = time.time()
-    sleep_time = 10
+    sleep_time = 3
     while status != "Job Completed":
         duration = time.time() - time_start
         job = job.api_get()

{ingestr-0.13.54 → ingestr-0.13.56}/ingestr/src/facebook_ads/settings.py

@@ -112,6 +112,8 @@ DEFAULT_INSIGHT_FIELDS = (
     "social_spend",
     "spend",
     "website_ctr",
+    "conversions",
+    "video_thruplay_watched_actions",
 )
 
 TInsightsLevels = Literal["account", "campaign", "adset", "ad"]

ingestr-0.13.56/ingestr/src/facebook_ads/utils.py

@@ -0,0 +1,39 @@
+from typing import Dict
+
+import dlt
+from dlt.common.configuration.inject import with_config
+from dlt.sources.helpers import requests
+
+
+@with_config(sections=("sources", "facebook_ads"))
+def debug_access_token(
+    access_token: str = dlt.secrets.value,
+    client_id: str = dlt.secrets.value,
+    client_secret: str = dlt.secrets.value,
+) -> str:
+    """Debugs the `access_token` providing info on expiration time, scopes etc. If arguments are not provides, `dlt` will inject them from configuration"""
+    debug_url = f"https://graph.facebook.com/debug_token?input_token={access_token}&access_token={client_id}|{client_secret}"
+    response = requests.get(debug_url)
+    data: Dict[str, str] = response.json()
+
+    if "error" in data:
+        raise Exception(f"Error debugging token: {data['error']}")
+
+    return data["data"]
+
+
+@with_config(sections=("sources", "facebook_ads"))
+def get_long_lived_token(
+    access_token: str = dlt.secrets.value,
+    client_id: str = dlt.secrets.value,
+    client_secret: str = dlt.secrets.value,
+) -> str:
+    """Gets the long lived access token (60 days) from `access_token`. If arguments are not provides, `dlt` will inject them from configuration"""
+    exchange_url = f"https://graph.facebook.com/v13.0/oauth/access_token?grant_type=fb_exchange_token&client_id={client_id}&client_secret={client_secret}&fb_exchange_token={access_token}"
+    response = requests.get(exchange_url)
+    data: Dict[str, str] = response.json()
+
+    if "error" in data:
+        raise Exception(f"Error refreshing token: {data['error']}")
+
+    return data["access_token"]

{ingestr-0.13.54 → ingestr-0.13.56}/ingestr/src/factory.py

@@ -64,6 +64,7 @@ from ingestr.src.sources import (
     SqlSource,
     StripeAnalyticsSource,
     TikTokSource,
+    TrustpilotSource,
     ZendeskSource,
 )
 
@@ -165,6 +166,7 @@ class SourceDestinationFactory:
         "pipedrive": PipedriveSource,
         "frankfurter": FrankfurterSource,
         "freshdesk": FreshdeskSource,
+        "trustpilot": TrustpilotSource,
         "phantombuster": PhantombusterSource,
         "elasticsearch": ElasticsearchSource,
         "attio": AttioSource,