ingestr 0.13.53__tar.gz → 0.13.55__tar.gz

{ingestr-0.13.53 → ingestr-0.13.55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.53
+Version: 0.13.55
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -52,14 +52,17 @@ Requires-Dist: dlt==1.10.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0
 Requires-Dist: duckdb==1.2.1
+Requires-Dist: ecdsa==0.19.1
 Requires-Dist: elastic-transport==8.17.1
 Requires-Dist: elasticsearch==8.10.1
+Requires-Dist: enum-compat==0.0.3
 Requires-Dist: et-xmlfile==2.0.0
 Requires-Dist: facebook-business==20.0.0
 Requires-Dist: filelock==3.17.0
 Requires-Dist: flatten-json==0.1.14
 Requires-Dist: frozenlist==1.5.0
 Requires-Dist: fsspec==2025.3.2
+Requires-Dist: future==1.0.0
 Requires-Dist: gcsfs==2025.3.2
 Requires-Dist: geojson==3.2.0
 Requires-Dist: gitdb==4.0.12
@@ -93,6 +96,7 @@ Requires-Dist: ibm-db-sa==0.4.1
 Requires-Dist: ibm-db==3.2.6
 Requires-Dist: idna==3.10
 Requires-Dist: inflection==0.5.1
+Requires-Dist: intuit-oauth==1.2.4
 Requires-Dist: isodate==0.7.2
 Requires-Dist: jmespath==1.0.1
 Requires-Dist: jsonpath-ng==1.7.0
@@ -147,8 +151,11 @@ Requires-Dist: pyparsing==3.2.1
 Requires-Dist: pyrate-limiter==3.7.0
 Requires-Dist: python-dateutil==2.9.0.post0
 Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: python-jose==3.5.0
+Requires-Dist: python-quickbooks==0.9.2
 Requires-Dist: pytz==2025.1
 Requires-Dist: pyyaml==6.0.2
+Requires-Dist: rauth==0.7.3
 Requires-Dist: redshift-connector==2.1.5
 Requires-Dist: requests-file==2.1.0
 Requires-Dist: requests-oauthlib==1.3.1

{ingestr-0.13.53 → ingestr-0.13.55}/docs/.vitepress/config.mjs

@@ -132,11 +132,13 @@ export default defineConfig({
               { text: "HubSpot", link: "/supported-sources/hubspot.md" },
               { text: "Klaviyo", link: "/supported-sources/klaviyo.md" },
               { text: "LinkedIn Ads", link: "/supported-sources/linkedin_ads.md" },
+              { text: "Mixpanel", link: "/supported-sources/mixpanel.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
               { text: "Personio", link: "/supported-sources/personio.md" },
               { text: "PhantomBuster", link: "/supported-sources/phantombuster.md" },
               { text: "Pipedrive", link: "/supported-sources/pipedrive.md" },
-              { text: "S3", link: "/supported-sources/s3.md"},
+              { text: "QuickBooks", link: "/supported-sources/quickbooks.md" },
+              { text: "S3", link: "/supported-sources/s3.md" },
               { text: "Salesforce", link: "/supported-sources/salesforce.md" },
               { text: "SFTP", link: "/supported-sources/sftp.md"},
               { text: "Shopify", link: "/supported-sources/shopify.md" },

ingestr-0.13.55/docs/supported-sources/facebook-ads.md

@@ -0,0 +1,202 @@
+# Facebook Ads
+
+Facebook Ads is the advertising platform that helps users to create targeted ads on Facebook, Instagram and Messenger.
+
+ingestr supports Facebook Ads as a source.
+
+## URI format
+
+The URI format for Facebook Ads is as follows:
+
+```plaintext
+facebookads://?access_token=<access_token>&account_id=<account_id>
+```
+
+URI parameters:
+
+- `access_token` is associated with Business Facebook App.
+- `account_id` is associated with Ad manager.
+
+Both are used for authentication with Facebook Ads API.
+
+The URI is used to connect to Facebook Ads API for extracting data.
+
+## Setting up a Facebook Ads Integration
+
+Facebook Ads requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/facebook_ads#setup-guide).
+
+Once you complete the guide, you should have an access token and an Account ID. Let's say your `access_token` is `abcdef` and `account_id` is `1234`, here's a sample command that will copy the data from Facebook Ads into a DuckDB database:
+
+```sh
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'campaigns' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.campaigns'
+```
+
+The result of this command will be a table in the `facebook.duckdb` database.
+
+## Tables
+
+Facebook Ads source allows ingesting the following sources into separate tables:
+
+- `campaigns`: Retrieves campaign data with fields:
+  - `id`
+  - `updated_time`
+  - `created_time`
+  - `name`
+  - `status`
+  - `effective_status`
+  - `objective`
+  - `start_time`
+  - `stop_time`
+  - `daily_budget`
+  - `lifetime_budget`
+
+- `ad_sets`: Retrieves ad set data with fields:
+  - `id`
+  - `updated_time`
+  - `created_time`
+  - `name`
+  - `status`
+  - `effective_status`
+  - `campaign_id`
+  - `start_time`
+  - `end_time`
+  - `daily_budget`
+  - `lifetime_budget`
+  - `optimization_goal`
+  - `promoted_object`
+  - `billing_event`
+  - `bid_amount`
+  - `bid_strategy`
+  - `targeting`
+
+- `leads`: Retrieves lead data with fields:
+  - `id`
+  - `created_time`
+  - `ad_id`
+  - `ad_name`
+  - `adset_id`
+  - `adset_name`
+  - `campaign_id`
+  - `campaign_name`
+  - `form_id`
+  - `field_data`
+
+- `ads_creatives`: Retrieves ad creative data with fields:
+  - `id`
+  - `name`
+  - `status`
+  - `thumbnail_url`
+  - `object_story_spec`
+  - `effective_object_story_id`
+  - `call_to_action_type`
+  - `object_type`
+  - `template_url`
+  - `url_tags`
+  - `instagram_actor_id`
+  - `product_set_id`
+
+- `ads`: Retrieves ad data with fields:
+  - `id`
+  - `updated_time`
+  - `created_time`
+  - `name`
+  - `status`
+  - `effective_status`
+  - `adset_id`
+  - `campaign_id`
+  - `creative`
+  - `targeting`
+  - `tracking_specs`
+  - `conversion_specs`
+
+- `facebook_insights`: Retrieves insights data with fields:
+  - `campaign_id`
+  - `adset_id`
+  - `ad_id`
+  - `date_start`
+  - `date_stop`
+  - `reach`
+  - `impressions`
+  - `frequency`
+  - `clicks`
+  - `unique_clicks`
+  - `ctr`
+  - `unique_ctr`
+  - `cpc`
+  - `cpm`
+  - `cpp`
+  - `spend`
+  - `actions`
+  - `action_values`
+  - `cost_per_action_type`
+  - `website_ctr`
+  - `account_currency`
+  - `ad_click_actions`
+  - `ad_name`
+  - `adset_name`
+  - `campaign_name`
+  - `country`
+  - `dma`
+  - `full_view_impressions`
+  - `full_view_reach`
+  - `inline_link_click_ctr`
+  - `outbound_clicks`
+  - `social_spend`
+  - `conversions`
+  - `video_thruplay_watched_actions`
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+### Facebook Insights Custom Configuration
+
+The `facebook_insights` table supports advanced configuration for breakdowns and custom metrics:
+
+#### Format Options
+
+1. **Default usage**: `facebook_insights`
+   - Uses default breakdown and default fields
+
+2. **Custom breakdown**: `facebook_insights:breakdown_type`
+   - Uses specified breakdown with default fields
+
+3. **Custom breakdown + metrics**: `facebook_insights:breakdown_type:metric1,metric2,metric3`
+   - Uses specified breakdown with custom metrics
+
+#### Available Breakdown Types
+
+- `ads_insights` (default)
+- `ads_insights_age_and_gender`
+- `ads_insights_country`
+- `ads_insights_platform_and_device`
+- `ads_insights_region`
+- `ads_insights_dma`
+- `ads_insights_hourly_advertiser`
+
+#### Examples
+
+```sh
+# Default facebook_insights
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.insights'
+
+# Age and gender breakdown with default metrics
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:ads_insights_age_and_gender' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.insights_demographics'
+
+# Country breakdown with custom metrics
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'facebook_insights:ads_insights_country:impressions,clicks,spend,reach,cpm,ctr' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.insights_by_country'
+```

ingestr-0.13.55/docs/supported-sources/mixpanel.md

@@ -0,0 +1,46 @@
+# Mixpanel
+
+[Mixpanel](https://mixpanel.com/) is an analytics service for tracking user interactions in web and mobile applications.
+
+ingestr supports Mixpanel as a source.
+
+## URI format
+
+```plaintext
+mixpanel://?username=<service_account_username>&password=<service_account_secret>&project_id=<project_id>&server=<server>
+```
+
+URI parameters:
+
+- `username`: Mixpanel service account username.
+- `password`: Mixpanel service account secret. This is the secret associated with the service account.
+- `project_id`: The numeric project ID.
+- `server`: (Optional) The server region to use. Can be "us", "eu", or "in". Defaults to "eu".
+
+
+To grab mixpanel credentials, please follow the guide [here](https://developer.mixpanel.com/reference/service-accounts).
+
+## Example
+
+Copy events from Mixpanel into a DuckDB database:
+
+```sh
+ingestr ingest \
+    --source-uri 'mixpanel://?username=my-service-account&password=my-secret&project_id=12345' \
+    --source-table 'events' \
+    --dest-uri duckdb:///mixpanel.duckdb \
+    --dest-table 'mixpanel.events'
+```
+
+
+<img alt="mixpanel" src="../media/mixpanel_ingestion.png"/>
+
+
+## Tables
+
+Mixpanel source allows ingesting the following tables:
+
+- `events`: Raw event data returned from the export API.
+- `profiles`: User profiles from the Engage API.
+
+Use these as `--source-table` values in the `ingestr ingest` command.

ingestr-0.13.55/docs/supported-sources/quickbooks.md

@@ -0,0 +1,49 @@
+# QuickBooks
+
+[QuickBooks](https://quickbooks.intuit.com/) is an accounting software package developed and marketed by Intuit.
+
+ingestr supports QuickBooks as a source.
+
+## URI format
+
+```plaintext
+quickbooks://?company_id=<company_id>client_id=<client_id>&client_secret=<client_secret>&refresh_token=<refresh_token>&access_token=<access_token>&environment=<environment>&minor_version=<minor_version>
+```
+
+URI parameters:
+- `company_id`: The QuickBooks company (realm) id.
+- `client_id`: OAuth client id from your Intuit application.
+- `client_secret`: OAuth client secret.
+- `refresh_token`: OAuth refresh token used to obtain access tokens.
+- `environment`: Optional environment name, either `production` or `sandbox`. Defaults to `production`.
+- `minor_version`: Optional API minor version.
+
+## Setting up a QuickBooks integration
+
+Follow Intuit's [OAuth setup guide](https://developer.intuit.com/app/developer/qbo/docs/develop/authentication-and-authorization) to create an app and generate your credentials.
+
+Once you have the credentials, you can ingest data. For example, to copy customers data into DuckDB:
+
+```sh
+ingestr ingest \
+--source-uri 'quickbooks://?company_id=1234567890&client_id=cid&client_secret=csecret&refresh_token=rtoken' \
+--source-table 'customers' \
+--dest-uri duckdb:///customers.duckdb \
+--dest-table 'dest.details'
+```
+This command will retrieve customers data and save it to the `dest.details` table in the DuckDB database.
+
+<img alt="quickbooks" src="../media/quickbook_ingestion.png"/>
+
+
+## Tables
+QuickBooks source allows ingesting the following tables:
+
+- `customers`: Retrives list of customers.
+- `invoices`: Retrives sales invoices.
+- `accounts`: Retrives details of accounts.
+- `vendors`: Retrives vendor records.
+- `payments`: Retrives payments recorded.
+
+Use these as the `--source-table` parameter in the `ingestr ingest` command.
+

ingestr-0.13.55/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.55"

{ingestr-0.13.53 → ingestr-0.13.55}/ingestr/src/destinations.py

@@ -476,7 +476,7 @@ class SqliteDestination(GenericSqlDestination):
 
     def dlt_run_params(self, uri: str, table: str, **kwargs):
         return {
-            #https://dlthub.com/docs/dlt-ecosystem/destinations/sqlalchemy#dataset-files
+            # https://dlthub.com/docs/dlt-ecosystem/destinations/sqlalchemy#dataset-files
             "dataset_name": "main",
             "table_name": table,
         }
@@ -495,6 +495,3 @@ class MySqlDestination(GenericSqlDestination):
             "dataset_name": database,
             "table_name": table,
         }
-
-
-

{ingestr-0.13.53 → ingestr-0.13.55}/ingestr/src/facebook_ads/__init__.py

@@ -116,6 +116,8 @@ def facebook_insights_source(
     batch_size: int = 50,
     request_timeout: int = 300,
     app_api_version: str = None,
+    start_date: pendulum.DateTime | None = None,
+    end_date: pendulum.DateTime | None = None,
 ) -> DltResource:
     """Incrementally loads insight reports with defined granularity level, fields, breakdowns etc.
 
@@ -148,27 +150,32 @@ def facebook_insights_source(
         account_id, access_token, request_timeout, app_api_version
     )
 
-    # we load with a defined lag
-    initial_load_start_date = pendulum.today().subtract(days=initial_load_past_days)
-    initial_load_start_date_str = initial_load_start_date.isoformat()
+    if start_date is None:
+        start_date = pendulum.today().subtract(days=initial_load_past_days)
+
+    columns = {}
+    for field in fields:
+        if field in INSIGHT_FIELDS_TYPES:
+            columns[field] = INSIGHT_FIELDS_TYPES[field]
 
     @dlt.resource(
         primary_key=INSIGHTS_PRIMARY_KEY,
         write_disposition="merge",
-        columns=INSIGHT_FIELDS_TYPES,
+        columns=columns,
     )
     def facebook_insights(
         date_start: dlt.sources.incremental[str] = dlt.sources.incremental(
             "date_start",
-            initial_value=initial_load_start_date_str,
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
             range_end="closed",
             range_start="closed",
+            lag=attribution_window_days_lag * 24 * 60 * 60,  # Convert days to seconds
         ),
     ) -> Iterator[TDataItems]:
-        start_date = get_start_date(date_start, attribution_window_days_lag)
+        start_date = get_start_date(date_start)
         end_date = pendulum.now()
 
-        # fetch insights in incremental day steps
         while start_date <= end_date:
             query = {
                 "level": level,
@@ -193,7 +200,10 @@ def facebook_insights_source(
                     }
                 ],
             }
-            job = execute_job(account.get_insights(params=query, is_async=True))
+            job = execute_job(
+                account.get_insights(params=query, is_async=True),
+                insights_max_async_sleep_seconds=10,
+            )
             yield list(map(process_report_item, job.get_result()))
             start_date = start_date.add(days=time_increment_days)
 

{ingestr-0.13.53 → ingestr-0.13.55}/ingestr/src/facebook_ads/helpers.py

@@ -31,14 +31,13 @@ from .settings import (
 
 def get_start_date(
     incremental_start_date: dlt.sources.incremental[str],
-    attribution_window_days_lag: int = 7,
 ) -> pendulum.DateTime:
     """
     Get the start date for incremental loading of Facebook Insights data.
     """
     start_date: pendulum.DateTime = ensure_pendulum_datetime(
         incremental_start_date.start_value
-    ).subtract(days=attribution_window_days_lag)
+    )
 
     # facebook forgets insights so trim the lag and warn
     min_start_date = pendulum.today().subtract(
@@ -65,7 +64,6 @@ def process_report_item(item: AbstractObject) -> DictStrAny:
     for pki in INSIGHTS_PRIMARY_KEY:
         if pki not in d:
             d[pki] = "no_" + pki
-
     return d
 
 
@@ -138,7 +136,7 @@ def execute_job(
 ) -> AbstractCrudObject:
     status: str = None
     time_start = time.time()
-    sleep_time = 10
+    sleep_time = 3
     while status != "Job Completed":
         duration = time.time() - time_start
         job = job.api_get()

{ingestr-0.13.53 → ingestr-0.13.55}/ingestr/src/facebook_ads/settings.py

@@ -112,6 +112,8 @@ DEFAULT_INSIGHT_FIELDS = (
     "social_spend",
     "spend",
     "website_ctr",
+    "conversions",
+    "video_thruplay_watched_actions",
 )
 
 TInsightsLevels = Literal["account", "campaign", "adset", "ad"]

ingestr-0.13.55/ingestr/src/facebook_ads/utils.py

@@ -0,0 +1,39 @@
+from typing import Dict
+
+import dlt
+from dlt.common.configuration.inject import with_config
+from dlt.sources.helpers import requests
+
+
+@with_config(sections=("sources", "facebook_ads"))
+def debug_access_token(
+    access_token: str = dlt.secrets.value,
+    client_id: str = dlt.secrets.value,
+    client_secret: str = dlt.secrets.value,
+) -> str:
+    """Debugs the `access_token` providing info on expiration time, scopes etc. If arguments are not provides, `dlt` will inject them from configuration"""
+    debug_url = f"https://graph.facebook.com/debug_token?input_token={access_token}&access_token={client_id}|{client_secret}"
+    response = requests.get(debug_url)
+    data: Dict[str, str] = response.json()
+
+    if "error" in data:
+        raise Exception(f"Error debugging token: {data['error']}")
+
+    return data["data"]
+
+
+@with_config(sections=("sources", "facebook_ads"))
+def get_long_lived_token(
+    access_token: str = dlt.secrets.value,
+    client_id: str = dlt.secrets.value,
+    client_secret: str = dlt.secrets.value,
+) -> str:
+    """Gets the long lived access token (60 days) from `access_token`. If arguments are not provides, `dlt` will inject them from configuration"""
+    exchange_url = f"https://graph.facebook.com/v13.0/oauth/access_token?grant_type=fb_exchange_token&client_id={client_id}&client_secret={client_secret}&fb_exchange_token={access_token}"
+    response = requests.get(exchange_url)
+    data: Dict[str, str] = response.json()
+
+    if "error" in data:
+        raise Exception(f"Error refreshing token: {data['error']}")
+
+    return data["access_token"]

{ingestr-0.13.53 → ingestr-0.13.55}/ingestr/src/factory.py

@@ -47,11 +47,13 @@ from ingestr.src.sources import (
     KlaviyoSource,
     LinkedInAdsSource,
     LocalCsvSource,
+    MixpanelSource,
     MongoDbSource,
     NotionSource,
     PersonioSource,
     PhantombusterSource,
     PipedriveSource,
+    QuickBooksSource,
     S3Source,
     SalesforceSource,
     SFTPSource,
@@ -140,6 +142,7 @@ class SourceDestinationFactory:
         "hubspot": HubspotSource,
         "airtable": AirtableSource,
         "klaviyo": KlaviyoSource,
+        "mixpanel": MixpanelSource,
         "appsflyer": AppsflyerSource,
         "kafka": KafkaSource,
         "adjust": AdjustSource,
@@ -166,6 +169,7 @@ class SourceDestinationFactory:
         "elasticsearch": ElasticsearchSource,
         "attio": AttioSource,
         "solidgate": SolidgateSource,
+        "quickbooks": QuickBooksSource,
         "smartsheet": SmartsheetSource,
         "sftp": SFTPSource,
     }

ingestr-0.13.55/ingestr/src/mixpanel/__init__.py

@@ -0,0 +1,62 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import MixpanelClient
+
+
+@dlt.source(max_table_nesting=0)
+def mixpanel_source(
+    username: str,
+    password: str,
+    project_id: str,
+    server: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    client = MixpanelClient(username, password, project_id, server)
+
+    @dlt.resource(write_disposition="merge", name="events", primary_key="distinct_id")
+    def events(
+        date=dlt.sources.incremental(
+            "time",
+            initial_value=start_date.int_timestamp,
+            end_value=end_date.int_timestamp if end_date else None,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if date.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = pendulum.from_timestamp(date.end_value)
+
+        start_dt = pendulum.from_timestamp(date.last_value)
+
+        yield from client.fetch_events(
+            start_dt,
+            end_dt,
+        )
+
+    @dlt.resource(write_disposition="merge", primary_key="distinct_id", name="profiles")
+    def profiles(
+        last_seen=dlt.sources.incremental(
+            "last_seen",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if last_seen.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = last_seen.end_value
+
+        start_dt = last_seen.last_value
+        yield from client.fetch_profiles(start_dt, end_dt)
+
+    return events, profiles