ingestr 0.7.7__tar.gz → 0.7.8__tar.gz

{ingestr-0.7.7 → ingestr-0.7.8}/.gitignore

@@ -13,4 +13,5 @@ pipeline_data
 dist
 docs/.vitepress/dist
 docs/.vitepress/cache
-node_modules
+node_modules
+*.duckdb

{ingestr-0.7.7 → ingestr-0.7.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.7.7
+Version: 0.7.8
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -14,11 +14,13 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
+Requires-Dist: confluent-kafka>=2.3.0
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: dlt==0.5.1
 Requires-Dist: duckdb-engine==0.11.5
 Requires-Dist: duckdb==0.10.2
+Requires-Dist: facebook-business==20.0.0
 Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: mysql-connector-python==9.0.0
@@ -185,6 +187,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>Chess.com</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>Facebook Ads</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Gorgias</td>
@@ -200,6 +207,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>HubSpot</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>Klaviyo</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Notion</td>

{ingestr-0.7.7 → ingestr-0.7.8}/README.md

@@ -139,6 +139,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>Chess.com</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>Facebook Ads</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Gorgias</td>
@@ -154,6 +159,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>HubSpot</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>Klaviyo</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Notion</td>

{ingestr-0.7.7 → ingestr-0.7.8}/docs/.vitepress/config.mjs

@@ -64,6 +64,7 @@ export default defineConfig({
                 text: "Google BigQuery",
                 link: "/supported-sources/bigquery.md",
               },
+              { text: "Kafka", link: "/supported-sources/kafka.md" },
               { text: "Local CSV Files", link: "/supported-sources/csv.md" },
               {
                 text: "Microsoft SQL Server",
@@ -85,9 +86,14 @@ export default defineConfig({
             items: [
               { text: "Airtable", link: "/supported-sources/airtable.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
+              {
+                text: "Facebook Ads",
+                link: "/supported-sources/facebook-ads.md",
+              },
               { text: "Google Sheets", link: "/supported-sources/gsheets.md" },
               { text: "Gorgias", link: "/supported-sources/gorgias.md" },
               { text: "HubSpot", link: "/supported-sources/hubspot.md" },
+              { text: "Klaviyo", link: "/supported-sources/klaviyo.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
               { text: "Shopify", link: "/supported-sources/shopify.md" },
               { text: "Slack", link: "/supported-sources/slack.md" },

ingestr-0.7.8/docs/supported-sources/facebook-ads.md

@@ -0,0 +1,51 @@
+# Facebook Ads
+
+Facebook Ads is the advertising platform that helps users to create targeted ads on Facebook, Instagram and Messenger.
+
+ingestr supports Facebook Ads as a source.
+
+## URI Format
+
+The URI format for Facebook Ads is as follows:
+
+```plaintext
+facebookads://?access_token=<access_token>&account_id=<account_id>
+```
+
+URI parameters:
+
+- `access_token` is associated with Business Facebook App.
+- `account_id` is associated with Ad manager.
+
+Both are used for authentication with Facebook Ads API.
+
+The URI is used to connect to Facebook Ads API for extracting data.
+
+## Setting up a Facebook Ads Integration
+
+Facebook Ads requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/facebook_ads#setup-guide).
+
+Once you complete the guide, you should have an Access_Token and Account ID . Let's say your access_token is `abcdef` and account_id is `1234` , here's a sample command that will copy the data from Facebook Ads into a duckdb database:
+
+```sh
+ingestr ingest \
+  --source-uri 'facebookads://?access_token=easdyh&account_id=1234' \
+  --source-table 'campaigns' \
+  --dest-uri 'duckdb:///facebook.duckdb' \
+  --dest-table 'dest.campaigns'
+```
+
+The result of this command will be a table in the `facebook.duckdb` database.
+
+## Available Tables
+
+Facebook Ads source allows ingesting the following sources into separate tables:
+
+- `campaigns`: Retrieves all DEFAULT_CAMPAIGN_FIELDS.
+- `ad_sets`: Retrieves all DEFAULT_ADSET_FIELDS.
+- `leads`: Retrieves all DEFAULT_LEAD_FIELDS.
+- `ads_creatives`: Retrieves all DEFAULT_ADCREATIVE_FIELDS.
+- `ads`: Retrieves all DEFAULT_ADS_FIELDS.
+- `facebook_insights`: Retrieves all DEFAULT_INSIGHTS_FIELDS.
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.

ingestr-0.7.8/docs/supported-sources/kafka.md

@@ -0,0 +1,38 @@
+# Apache Kafka
+[Apache Kafka](https://kafka.apache.org/) is a distributed event streaming platform used by thousands of companies for high-performance data pipelines, streaming analytics, data integration, and mission-critical applications.
+
+ingestr supports Apache Kafka as a source.
+
+## URI Format
+The URI format for Apache Kafka is as follows:
+
+```plaintext
+kafka://?bootstrap_servers=localhost:9092&group_id=test_group&security_protocol=SASL_SSL&sasl_mechanisms=PLAIN&sasl_username=example_username&sasl_password=example_secret&batch_size=1000&batch_timeout=3
+```
+
+URI parameters:
+- `bootstrap_servers`: The Kafka server(s) to connect to, typically in the form of a host and port (e.g., `localhost:9092`).
+- `group_id`: The consumer group ID used for identifying the client when consuming messages.
+- `security_protocol`: The protocol used to communicate with brokers (e.g., `SASL_SSL` for secure communication).
+- `sasl_mechanisms`: The SASL mechanism to be used for authentication (e.g., `PLAIN`).
+- `sasl_username`: The username for SASL authentication.
+- `sasl_password`: The password for SASL authentication.
+- `batch_size`: The number of messages to fetch in a single batch, defaults to 3000.
+- `batch_timeout`: The maximum time to wait for messages, defaults to 3 seconds.
+
+The URI is used to connect to the Kafka brokers for ingesting messages.
+
+### Group ID
+The group ID is used to identify the consumer group that reads messages from a topic. Kafka uses the group ID to manage consumer offsets and assign partitions to consumers, which means that the group ID is the key to reading messages from the correct partition and position in the topic.
+
+Once you have your Kafka server, credentials, and group ID set up, here's a sample command to ingest messages from a Kafka topic into a duckdb database:
+
+```sh
+ingestr ingest \
+    --source-uri 'kafka://?bootstrap_servers=localhost:9092' \
+    --source-table 'my-topic' \
+    --dest-uri duckdb:///kafka.duckdb \
+    --dest-table 'kafka.my_topic'
+```
+
+The result of this command will be a table in the `kafka.duckdb` database with JSON columns.

ingestr-0.7.8/docs/supported-sources/klaviyo.md

@@ -0,0 +1,64 @@
+# Klaviyo
+
+[Klaviyo](https://www.klaviyo.com/) is a marketing automation platform that helps businesses build and manage smarter digital relationships with their customers by connecting through personalized email and enhancing customer loyality.
+
+ingestr supports Klaviyo as a source.
+
+## URI Format
+
+The URI format for Klaviyo is as follows:
+
+```plaintext
+klaviyo://?api_key=<api-key>
+```
+
+URI parameters:
+
+- `api_key`: The API key used for authentication with the Klaviyo API.
+
+The URI is used to connect to the Klaviyo API for extracting data.
+
+```bash
+ingestr ingest --source-table 'events' --source-uri 'klaviyo://?api_key=pk_test' --dest-uri duckdb:///klaviyo.duckdb --interval-start 2022-01-01 --dest-table 'klaviyo.events' --extract-parallelism 20
+```
+
+This command fethes all the events that are created/updated since 2022-01-01 and writes them to `klaviyo.events` table on DuckDB, using 20 parallel threads to improve performance and efficiently handle large data .
+
+## Available Tables
+
+Klaviyo source allows ingesting the following sources into separate tables:
+
+[events](https://developers.klaviyo.com/en/reference/events_api_overview): Retrieves all events in an account where each event represents an action taken by a profile such as a password reset or a product order.
+
+[profiles](https://developers.klaviyo.com/en/reference/profiles_api_overview): Retrieves all profiles in an account where each profile includes details like organization, job title, email and other attributes.
+
+[campaigns](https://developers.klaviyo.com/en/reference/campaigns_api_overview): Retrieves all campaigns in an account where each campaign is a targeted message sent to a specific audience.
+
+[metrics](https://developers.klaviyo.com/en/reference/metrics_api_overview): Retrieves all metrics in an account where each metric represents a category of events or actions a person can take.
+
+[tags](https://developers.klaviyo.com/en/reference/get_tags): Retrieves all tags in an account.
+
+[coupons](https://developers.klaviyo.com/en/reference/get_coupons): Retrieves all coupons in an account.
+
+[catalog-variants](https://developers.klaviyo.com/en/reference/get_catalog_variants): Retrieves all variants in an account.
+
+[catalog-categories](https://developers.klaviyo.com/en/reference/get_catalog_categories): Retrieves all catalog categories in an account.
+
+[catalog-items](https://developers.klaviyo.com/en/reference/get_catalog_items): Retrieves all catalog items in an account.
+
+[flows](https://developers.klaviyo.com/en/reference/get_flows): Retrieves all flows in an account where flow is a sequence of automated actions that is triggered when a person performs a specific action.
+
+[lists](https://developers.klaviyo.com/en/reference/get_lists): Retrieves all lists in an account.
+
+[images](https://developers.klaviyo.com/en/reference/get_images): Retrieves all images in an account..
+
+[segments](https://developers.klaviyo.com/en/reference/get_segments): Retrieves all segments in an account where segment is a dynamic list that contains profiles meeting a certain set of conditions.
+
+[forms](https://developers.klaviyo.com/en/reference/get_forms): Retrieves all forms in an account.
+
+[templates](https://developers.klaviyo.com/en/reference/get_templates): Retrieves all templates in an account.
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+> [!WARNING]
+> Klaviyo does not support incremental loading for many endpoints in its APIs, which means ingestr will load endpoints incrementally if they support it, and do a full-refresh if not.

{ingestr-0.7.7 → ingestr-0.7.8}/ingestr/main.py

@@ -244,6 +244,13 @@ def ingest(
             envvar="PIPELINES_DIR",
         ),
     ] = None,  # type: ignore
+    extract_parallelism: Annotated[
+        Optional[int],
+        typer.Option(
+            help="The number of parallel jobs to run for extracting data from the source, only applicable for certain sources",
+            envvar="EXTRACT_PARALLELISM",
+        ),
+    ] = 5,  # type: ignore
 ):
     track(
         "command_triggered",
@@ -253,6 +260,8 @@ def ingest(
     )
 
     dlt.config["data_writer.file_max_items"] = loader_file_size
+    dlt.config["extract.workers"] = extract_parallelism
+    dlt.config["extract.max_parallel_items"] = extract_parallelism
     if schema_naming != SchemaNaming.default:
         dlt.config["schema.naming"] = schema_naming.value
 

ingestr-0.7.8/ingestr/src/.gitignore

@@ -0,0 +1,10 @@
+# ignore secrets, virtual environments and typical python compilation artifacts
+secrets.toml
+# ignore basic python artifacts
+.env
+**/__pycache__/
+**/*.py[cod]
+**/*$py.class
+# ignore duckdb
+*.duckdb
+*.wal

ingestr-0.7.8/ingestr/src/facebook_ads/__init__.py

@@ -0,0 +1,197 @@
+"""Loads campaigns, ads sets, ads, leads and insight data from Facebook Marketing API"""
+
+from typing import Iterator, Sequence
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.typing import TDataItems
+from dlt.sources import DltResource
+from facebook_business.adobjects.ad import Ad
+
+from .helpers import (
+    execute_job,
+    get_ads_account,
+    get_data_chunked,
+    get_start_date,
+    process_report_item,
+)
+from .settings import (
+    ALL_ACTION_ATTRIBUTION_WINDOWS,
+    ALL_ACTION_BREAKDOWNS,
+    DEFAULT_AD_FIELDS,
+    DEFAULT_ADCREATIVE_FIELDS,
+    DEFAULT_ADSET_FIELDS,
+    DEFAULT_CAMPAIGN_FIELDS,
+    DEFAULT_INSIGHT_FIELDS,
+    DEFAULT_LEAD_FIELDS,
+    INSIGHT_FIELDS_TYPES,
+    INSIGHTS_BREAKDOWNS_OPTIONS,
+    INSIGHTS_PRIMARY_KEY,
+    INVALID_INSIGHTS_FIELDS,
+    TInsightsBreakdownOptions,
+    TInsightsLevels,
+)
+
+
+@dlt.source(name="facebook_ads", max_table_nesting=0)
+def facebook_ads_source(
+    account_id: str = dlt.config.value,
+    access_token: str = dlt.secrets.value,
+    chunk_size: int = 50,
+    request_timeout: float = 300.0,
+    app_api_version: str = "v20.0",
+) -> Sequence[DltResource]:
+    """Returns a list of resources to load campaigns, ad sets, ads, creatives and ad leads data from Facebook Marketing API.
+
+    All the resources have `replace` write disposition by default and define primary keys. Resources are parametrized and allow the user
+    to change the set of fields that will be loaded from the API and the object statuses that will be loaded. See the demonstration script for details.
+
+    You can convert the source into merge resource to keep the deleted objects. Currently Marketing API does not return deleted objects. See the demo script.
+
+    We also provide a transformation `enrich_ad_objects` that you can add to any of the resources to get additional data per object via `object.get_api`
+
+    Args:
+        account_id (str, optional): Account id associated with add manager. See README.md
+        access_token (str, optional): Access token associated with the Business Facebook App. See README.md
+        chunk_size (int, optional): A size of the page and batch request. You may need to decrease it if you request a lot of fields. Defaults to 50.
+        request_timeout (float, optional): Connection timeout. Defaults to 300.0.
+        app_api_version(str, optional): A version of the facebook api required by the app for which the access tokens were issued ie. 'v17.0'. Defaults to the facebook_business library default version
+
+    Returns:
+        Sequence[DltResource]: campaigns, ads, ad_sets, ad_creatives, leads
+    """
+    account = get_ads_account(
+        account_id, access_token, request_timeout, app_api_version
+    )
+
+    @dlt.resource(primary_key="id", write_disposition="replace")
+    def campaigns(
+        fields: Sequence[str] = DEFAULT_CAMPAIGN_FIELDS, states: Sequence[str] = None
+    ) -> Iterator[TDataItems]:
+        yield get_data_chunked(account.get_campaigns, fields, states, chunk_size)
+
+    @dlt.resource(primary_key="id", write_disposition="replace")
+    def ads(
+        fields: Sequence[str] = DEFAULT_AD_FIELDS, states: Sequence[str] = None
+    ) -> Iterator[TDataItems]:
+        yield get_data_chunked(account.get_ads, fields, states, chunk_size)
+
+    @dlt.resource(primary_key="id", write_disposition="replace")
+    def ad_sets(
+        fields: Sequence[str] = DEFAULT_ADSET_FIELDS, states: Sequence[str] = None
+    ) -> Iterator[TDataItems]:
+        yield get_data_chunked(account.get_ad_sets, fields, states, chunk_size)
+
+    @dlt.transformer(primary_key="id", write_disposition="replace", selected=True)
+    def leads(
+        items: TDataItems,
+        fields: Sequence[str] = DEFAULT_LEAD_FIELDS,
+        states: Sequence[str] = None,
+    ) -> Iterator[TDataItems]:
+        for item in items:
+            ad = Ad(item["id"])
+            yield get_data_chunked(ad.get_leads, fields, states, chunk_size)
+
+    @dlt.resource(primary_key="id", write_disposition="replace")
+    def ad_creatives(
+        fields: Sequence[str] = DEFAULT_ADCREATIVE_FIELDS, states: Sequence[str] = None
+    ) -> Iterator[TDataItems]:
+        yield get_data_chunked(account.get_ad_creatives, fields, states, chunk_size)
+
+    return campaigns, ads, ad_sets, ad_creatives, ads | leads
+
+
+@dlt.source(name="facebook_ads", max_table_nesting=0)
+def facebook_insights_source(
+    account_id: str = dlt.config.value,
+    access_token: str = dlt.secrets.value,
+    initial_load_past_days: int = 1,
+    fields: Sequence[str] = DEFAULT_INSIGHT_FIELDS,
+    attribution_window_days_lag: int = 7,
+    time_increment_days: int = 1,
+    breakdowns: TInsightsBreakdownOptions = "ads_insights",
+    action_breakdowns: Sequence[str] = ALL_ACTION_BREAKDOWNS,
+    level: TInsightsLevels = "ad",
+    action_attribution_windows: Sequence[str] = ALL_ACTION_ATTRIBUTION_WINDOWS,
+    batch_size: int = 50,
+    request_timeout: int = 300,
+    app_api_version: str = None,
+) -> DltResource:
+    """Incrementally loads insight reports with defined granularity level, fields, breakdowns etc.
+
+    By default, the reports are generated one by one for each day, starting with today - attribution_window_days_lag. On subsequent runs, only the reports
+    from the last report date until today are loaded (incremental load). The reports from last 7 days (`attribution_window_days_lag`) are refreshed on each load to
+    account for changes during attribution window.
+
+    Mind that each report is a job and takes some time to execute.
+
+    Args:
+        account_id: str = dlt.config.value,
+        access_token: str = dlt.secrets.value,
+        initial_load_past_days (int, optional): How many past days (starting from today) to intially load. Defaults to 30.
+        fields (Sequence[str], optional): A list of fields to include in each reports. Note that `breakdowns` option adds fields automatically. Defaults to DEFAULT_INSIGHT_FIELDS.
+        attribution_window_days_lag (int, optional): Attribution window in days. The reports in attribution window are refreshed on each run.. Defaults to 7.
+        time_increment_days (int, optional): The report aggregation window in days. use 7 for weekly aggregation. Defaults to 1.
+        breakdowns (TInsightsBreakdownOptions, optional): A presents with common aggregations. See settings.py for details. Defaults to "ads_insights_age_and_gender".
+        action_breakdowns (Sequence[str], optional): Action aggregation types. See settings.py for details. Defaults to ALL_ACTION_BREAKDOWNS.
+        level (TInsightsLevels, optional): The granularity level. Defaults to "ad".
+        action_attribution_windows (Sequence[str], optional): Attribution windows for actions. Defaults to ALL_ACTION_ATTRIBUTION_WINDOWS.
+        batch_size (int, optional): Page size when reading data from particular report. Defaults to 50.
+        request_timeout (int, optional): Connection timeout. Defaults to 300.
+        app_api_version(str, optional): A version of the facebook api required by the app for which the access tokens were issued ie. 'v17.0'. Defaults to the facebook_business library default version
+
+    Returns:
+        DltResource: facebook_insights
+
+    """
+    account = get_ads_account(
+        account_id, access_token, request_timeout, app_api_version
+    )
+
+    # we load with a defined lag
+    initial_load_start_date = pendulum.today().subtract(days=initial_load_past_days)
+    initial_load_start_date_str = initial_load_start_date.isoformat()
+
+    @dlt.resource(
+        primary_key=INSIGHTS_PRIMARY_KEY,
+        write_disposition="merge",
+        columns=INSIGHT_FIELDS_TYPES,
+    )
+    def facebook_insights(
+        date_start: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "date_start", initial_value=initial_load_start_date_str
+        ),
+    ) -> Iterator[TDataItems]:
+        start_date = get_start_date(date_start, attribution_window_days_lag)
+        end_date = pendulum.now()
+
+        # fetch insights in incremental day steps
+        while start_date <= end_date:
+            query = {
+                "level": level,
+                "action_breakdowns": list(action_breakdowns),
+                "breakdowns": list(
+                    INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["breakdowns"]
+                ),
+                "limit": batch_size,
+                "fields": list(
+                    set(fields)
+                    .union(INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["fields"])
+                    .difference(INVALID_INSIGHTS_FIELDS)
+                ),
+                "time_increment": time_increment_days,
+                "action_attribution_windows": list(action_attribution_windows),
+                "time_ranges": [
+                    {
+                        "since": start_date.to_date_string(),
+                        "until": start_date.add(
+                            days=time_increment_days - 1
+                        ).to_date_string(),
+                    }
+                ],
+            }
+            job = execute_job(account.get_insights(params=query, is_async=True))
+            yield list(map(process_report_item, job.get_result()))
+            start_date = start_date.add(days=time_increment_days)
+
+    return facebook_insights

ingestr-0.7.8/ingestr/src/facebook_ads/exceptions.py

@@ -0,0 +1,5 @@
+from dlt.extract.exceptions import DltResourceException
+
+
+class InsightsJobTimeout(DltResourceException):
+    pass