ingestr 0.13.2__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/elasticsearch/helpers.py

@@ -0,0 +1,138 @@
+"""Elasticsearch destination helpers"""
+
+import json
+import logging
+from typing import Any, Dict, Iterator
+from urllib.parse import urlparse
+
+import dlt
+
+from elasticsearch import Elasticsearch
+from elasticsearch.helpers import bulk
+
+# Suppress Elasticsearch transport logging
+logging.getLogger("elasticsearch.transport").setLevel(logging.WARNING)
+logging.getLogger("elastic_transport.transport").setLevel(logging.WARNING)
+
+
+def process_file_items(file_path: str) -> Iterator[Dict[str, Any]]:
+    """Process items from a file path (JSONL format)."""
+    with open(file_path, "r") as f:
+        for line in f:
+            if line.strip():
+                doc = json.loads(line.strip())
+                # Clean DLT metadata
+                cleaned_doc = {
+                    k: v for k, v in doc.items() if not k.startswith("_dlt_")
+                }
+                yield cleaned_doc
+
+
+def process_iterable_items(items: Any) -> Iterator[Dict[str, Any]]:
+    """Process items from an iterable."""
+    for item in items:
+        if isinstance(item, dict):
+            # Clean DLT metadata
+            cleaned_item = {k: v for k, v in item.items() if not k.startswith("_dlt_")}
+            yield cleaned_item
+
+
+@dlt.destination(
+    name="elasticsearch",
+    loader_file_format="typed-jsonl",
+    batch_size=1000,
+    naming_convention="snake_case",
+)
+def elasticsearch_insert(
+    items, table, connection_string: str = dlt.secrets.value
+) -> None:
+    """Insert data into Elasticsearch index.
+
+    Args:
+        items: Data items (file path or iterable)
+        table: Table metadata containing name and schema info
+        connection_string: Elasticsearch connection string
+    """
+    # Parse connection string
+    parsed = urlparse(connection_string)
+
+    # Build Elasticsearch client configuration
+    actual_url = connection_string
+    secure = True  # Default to HTTPS (secure by default)
+
+    if connection_string.startswith("elasticsearch://"):
+        actual_url = connection_string.replace("elasticsearch://", "")
+
+        # Parse to check for query parameters
+        temp_parsed = urlparse("http://" + actual_url)
+        from urllib.parse import parse_qs
+
+        query_params = parse_qs(temp_parsed.query)
+
+        # Check ?secure parameter (defaults to true)
+        if "secure" in query_params:
+            secure = query_params["secure"][0].lower() in ["true", "1", "yes"]
+
+        # Remove query params from URL for ES client
+        actual_url = actual_url.split("?")[0]
+
+        # Add scheme
+        scheme = "https" if secure else "http"
+        actual_url = f"{scheme}://{actual_url}"
+
+        parsed = urlparse(actual_url)
+
+    es_config: Dict[str, Any] = {
+        "hosts": [actual_url],
+        "verify_certs": secure,
+        "ssl_show_warn": False,
+    }
+
+    # Add authentication if present
+    if parsed.username and parsed.password:
+        es_config["http_auth"] = (parsed.username, parsed.password)
+
+    # Get index name from table metadata
+    index_name = table["name"]
+
+    # Connect to Elasticsearch
+    client = Elasticsearch(**es_config)
+
+    # Replace mode: delete existing index if it exists
+    if client.indices.exists(index=index_name):
+        client.indices.delete(index=index_name)
+
+    # Process and insert documents
+    if isinstance(items, str):
+        documents = process_file_items(items)
+    else:
+        documents = process_iterable_items(items)
+
+    # Prepare documents for bulk insert as generator
+    def doc_generator():
+        for doc in documents:
+            es_doc: Dict[str, Any] = {"_index": index_name, "_source": doc.copy()}
+
+            # Use _id if present, otherwise let ES generate one
+            if "_id" in doc:
+                es_doc["_id"] = str(doc["_id"])
+                # Remove _id from source since it's metadata
+                if "_id" in es_doc["_source"]:
+                    del es_doc["_source"]["_id"]
+            elif "id" in doc:
+                es_doc["_id"] = str(doc["id"])
+
+            yield es_doc
+
+    # Bulk insert
+    try:
+        _, failed_items = bulk(client, doc_generator(), request_timeout=60)
+        if failed_items:
+            failed_count = (
+                len(failed_items) if isinstance(failed_items, list) else failed_items
+            )
+            raise Exception(
+                f"Failed to insert {failed_count} documents: {failed_items}"
+            )
+    except Exception as e:
+        raise Exception(f"Elasticsearch bulk insert failed: {str(e)}")

ingestr/src/errors.py

@@ -1,3 +1,6 @@
+import requests
+
+
 class MissingValueError(Exception):
     def __init__(self, value, source):
         super().__init__(f"{value} is required to connect to {source}")
@@ -16,3 +19,8 @@ class InvalidBlobTableError(Exception):
             f"Invalid source table for {source} "
             "Ensure that the table is in the format {bucket-name}/{file glob}"
         )
+
+
+class HTTPError(Exception):
+    def __init__(self, source: requests.HTTPError):
+        super().__init__(f"HTTP {source.response.status_code}: {source.response.text}")

ingestr/src/facebook_ads/__init__.py

@@ -4,6 +4,7 @@ from typing import Iterator, Sequence
 
 import dlt
 from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import TDataItems
 from dlt.sources import DltResource
 from facebook_business.adobjects.ad import Ad
@@ -12,7 +13,6 @@ from .helpers import (
     execute_job,
     get_ads_account,
     get_data_chunked,
-    get_start_date,
     process_report_item,
 )
 from .settings import (
@@ -22,13 +22,8 @@ from .settings import (
     DEFAULT_ADCREATIVE_FIELDS,
     DEFAULT_ADSET_FIELDS,
     DEFAULT_CAMPAIGN_FIELDS,
-    DEFAULT_INSIGHT_FIELDS,
     DEFAULT_LEAD_FIELDS,
     INSIGHT_FIELDS_TYPES,
-    INSIGHTS_BREAKDOWNS_OPTIONS,
-    INSIGHTS_PRIMARY_KEY,
-    INVALID_INSIGHTS_FIELDS,
-    TInsightsBreakdownOptions,
     TInsightsLevels,
 )
 
@@ -106,16 +101,20 @@ def facebook_insights_source(
     account_id: str = dlt.config.value,
     access_token: str = dlt.secrets.value,
     initial_load_past_days: int = 1,
-    fields: Sequence[str] = DEFAULT_INSIGHT_FIELDS,
-    attribution_window_days_lag: int = 7,
+    dimensions: Sequence[str] = None,
+    fields: Sequence[str] = None,
     time_increment_days: int = 1,
-    breakdowns: TInsightsBreakdownOptions = "ads_insights",
     action_breakdowns: Sequence[str] = ALL_ACTION_BREAKDOWNS,
     level: TInsightsLevels = "ad",
     action_attribution_windows: Sequence[str] = ALL_ACTION_ATTRIBUTION_WINDOWS,
     batch_size: int = 50,
     request_timeout: int = 300,
     app_api_version: str = None,
+    start_date: pendulum.DateTime | None = None,
+    end_date: pendulum.DateTime | None = None,
+    insights_max_wait_to_finish_seconds: int = 60 * 60 * 4,
+    insights_max_wait_to_start_seconds: int = 60 * 30,
+    insights_max_async_sleep_seconds: int = 20,
 ) -> DltResource:
     """Incrementally loads insight reports with defined granularity level, fields, breakdowns etc.
 
@@ -148,40 +147,54 @@ def facebook_insights_source(
         account_id, access_token, request_timeout, app_api_version
     )
 
-    # we load with a defined lag
-    initial_load_start_date = pendulum.today().subtract(days=initial_load_past_days)
-    initial_load_start_date_str = initial_load_start_date.isoformat()
+    if start_date is None:
+        start_date = pendulum.today().subtract(days=initial_load_past_days)
+
+    if dimensions is None:
+        dimensions = []
+    if fields is None:
+        fields = []
+
+    columns = {}
+    for field in fields:
+        if field in INSIGHT_FIELDS_TYPES:
+            columns[field] = INSIGHT_FIELDS_TYPES[field]
 
     @dlt.resource(
-        primary_key=INSIGHTS_PRIMARY_KEY,
         write_disposition="merge",
-        columns=INSIGHT_FIELDS_TYPES,
+        merge_key="date_start",
+        columns=columns,
     )
     def facebook_insights(
         date_start: dlt.sources.incremental[str] = dlt.sources.incremental(
             "date_start",
-            initial_value=initial_load_start_date_str,
+            initial_value=ensure_pendulum_datetime(start_date).start_of("day").date(),
+            end_value=ensure_pendulum_datetime(end_date).end_of("day").date()
+            if end_date
+            else None,
             range_end="closed",
             range_start="closed",
         ),
     ) -> Iterator[TDataItems]:
-        start_date = get_start_date(date_start, attribution_window_days_lag)
-        end_date = pendulum.now()
+        start_date = date_start.last_value
+        if date_start.end_value:
+            end_date_val = pendulum.instance(date_start.end_value)
+
+            end_date = (
+                end_date_val
+                if isinstance(end_date_val, pendulum.Date)
+                else end_date_val.date()
+            )
+        else:
+            end_date = pendulum.now().date()
 
-        # fetch insights in incremental day steps
         while start_date <= end_date:
             query = {
                 "level": level,
                 "action_breakdowns": list(action_breakdowns),
-                "breakdowns": list(
-                    INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["breakdowns"]
-                ),
+                "breakdowns": dimensions,
                 "limit": batch_size,
-                "fields": list(
-                    set(fields)
-                    .union(INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["fields"])
-                    .difference(INVALID_INSIGHTS_FIELDS)
-                ),
+                "fields": fields,
                 "time_increment": time_increment_days,
                 "action_attribution_windows": list(action_attribution_windows),
                 "time_ranges": [
@@ -193,8 +206,14 @@ def facebook_insights_source(
                     }
                 ],
             }
-            job = execute_job(account.get_insights(params=query, is_async=True))
-            yield list(map(process_report_item, job.get_result()))
+            job = execute_job(
+                account.get_insights(params=query, is_async=True),
+                insights_max_async_sleep_seconds=insights_max_async_sleep_seconds,
+                insights_max_wait_to_finish_seconds=insights_max_wait_to_finish_seconds,
+                insights_max_wait_to_start_seconds=insights_max_wait_to_start_seconds,
+            )
+            output = list(map(process_report_item, job.get_result()))
+            yield output
             start_date = start_date.add(days=time_increment_days)
 
     return facebook_insights

ingestr/src/facebook_ads/helpers.py

@@ -3,14 +3,13 @@
 import functools
 import itertools
 import time
+from datetime import datetime
 from typing import Any, Iterator, Sequence
 
-import dlt
 import humanize
 import pendulum
 from dlt.common import logger
 from dlt.common.configuration.inject import with_config
-from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import DictStrAny, TDataItem, TDataItems
 from dlt.sources.helpers import requests
 from dlt.sources.helpers.requests import Client
@@ -23,49 +22,21 @@ from facebook_business.api import FacebookResponse
 
 from .exceptions import InsightsJobTimeout
 from .settings import (
-    FACEBOOK_INSIGHTS_RETENTION_PERIOD,
     INSIGHTS_PRIMARY_KEY,
     TFbMethod,
 )
 
 
-def get_start_date(
-    incremental_start_date: dlt.sources.incremental[str],
-    attribution_window_days_lag: int = 7,
-) -> pendulum.DateTime:
-    """
-    Get the start date for incremental loading of Facebook Insights data.
-    """
-    start_date: pendulum.DateTime = ensure_pendulum_datetime(
-        incremental_start_date.start_value
-    ).subtract(days=attribution_window_days_lag)
-
-    # facebook forgets insights so trim the lag and warn
-    min_start_date = pendulum.today().subtract(
-        months=FACEBOOK_INSIGHTS_RETENTION_PERIOD
-    )
-    if start_date < min_start_date:
-        logger.warning(
-            "%s: Start date is earlier than %s months ago, using %s instead. "
-            "For more information, see https://www.facebook.com/business/help/1695754927158071?id=354406972049255",
-            "facebook_insights",
-            FACEBOOK_INSIGHTS_RETENTION_PERIOD,
-            min_start_date,
-        )
-        start_date = min_start_date
-        incremental_start_date.start_value = min_start_date
-
-    # lag the incremental start date by attribution window lag
-    incremental_start_date.start_value = start_date.isoformat()
-    return start_date
-
-
 def process_report_item(item: AbstractObject) -> DictStrAny:
+    if "date_start" in item:
+        item["date_start"] = datetime.strptime(item["date_start"], "%Y-%m-%d").date()
+    if "date_stop" in item:
+        item["date_stop"] = datetime.strptime(item["date_stop"], "%Y-%m-%d").date()
+
     d: DictStrAny = item.export_all_data()
     for pki in INSIGHTS_PRIMARY_KEY:
         if pki not in d:
             d[pki] = "no_" + pki
-
     return d
 
 
@@ -138,17 +109,22 @@ def execute_job(
 ) -> AbstractCrudObject:
     status: str = None
     time_start = time.time()
-    sleep_time = 10
+    sleep_time = 3
     while status != "Job Completed":
+        print("-----")
+        print("waiting for job to finish")
         duration = time.time() - time_start
         job = job.api_get()
         status = job["async_status"]
         percent_complete = job["async_percent_completion"]
+        print("async_status", status)
+        print("percent_complete", percent_complete)
 
         job_id = job["id"]
         logger.info("%s, %d%% done", status, percent_complete)
 
         if status == "Job Completed":
+            print("job completed")
             return job
 
         if duration > insights_max_wait_to_start_seconds and percent_complete == 0:
@@ -168,7 +144,7 @@ def execute_job(
             raise InsightsJobTimeout(
                 "facebook_insights",
                 pretty_error_message.format(
-                    job_id, insights_max_wait_to_finish_seconds // 60
+                    job_id, insights_max_wait_to_finish_seconds
                 ),
             )
 
@@ -253,3 +229,49 @@ def notify_on_token_expiration(access_token_expires_at: int = None) -> None:
             logger.error(
                 f"Access Token expires in {humanize.precisedelta(pendulum.now() - expires_at)}. Replace the token now!"
             )
+
+
+def parse_insights_table_to_source_kwargs(table: str) -> DictStrAny:
+    import typing
+
+    from ingestr.src.facebook_ads.settings import (
+        INSIGHTS_BREAKDOWNS_OPTIONS,
+        TInsightsBreakdownOptions,
+        TInsightsLevels,
+    )
+
+    parts = table.split(":")
+
+    source_kwargs = {}
+
+    breakdown_type = parts[1]
+
+    valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
+    if breakdown_type in valid_breakdowns:
+        dimensions = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["breakdowns"]
+        fields = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["fields"]
+        source_kwargs["dimensions"] = dimensions
+        source_kwargs["fields"] = fields
+    else:
+        dimensions = breakdown_type.split(",")
+        valid_levels = list(typing.get_args(TInsightsLevels))
+        level = None
+        for valid_level in reversed(valid_levels):
+            if valid_level in dimensions:
+                level = valid_level
+                dimensions.remove(valid_level)
+                break
+
+        source_kwargs["level"] = level
+        source_kwargs["dimensions"] = dimensions
+
+    # If custom metrics are provided, parse them
+    if len(parts) == 3:
+        fields = [f.strip() for f in parts[2].split(",") if f.strip()]
+        if not fields:
+            raise ValueError(
+                "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
+            )
+        source_kwargs["fields"] = fields
+
+    return source_kwargs

ingestr/src/facebook_ads/settings.py

@@ -112,6 +112,8 @@ DEFAULT_INSIGHT_FIELDS = (
     "social_spend",
     "spend",
     "website_ctr",
+    "conversions",
+    "video_thruplay_watched_actions",
 )
 
 TInsightsLevels = Literal["account", "campaign", "adset", "ad"]

ingestr/src/facebook_ads/utils.py

@@ -0,0 +1,39 @@
+from typing import Dict
+
+import dlt
+from dlt.common.configuration.inject import with_config
+from dlt.sources.helpers import requests
+
+
+@with_config(sections=("sources", "facebook_ads"))
+def debug_access_token(
+    access_token: str = dlt.secrets.value,
+    client_id: str = dlt.secrets.value,
+    client_secret: str = dlt.secrets.value,
+) -> str:
+    """Debugs the `access_token` providing info on expiration time, scopes etc. If arguments are not provides, `dlt` will inject them from configuration"""
+    debug_url = f"https://graph.facebook.com/debug_token?input_token={access_token}&access_token={client_id}|{client_secret}"
+    response = requests.get(debug_url)
+    data: Dict[str, str] = response.json()
+
+    if "error" in data:
+        raise Exception(f"Error debugging token: {data['error']}")
+
+    return data["data"]
+
+
+@with_config(sections=("sources", "facebook_ads"))
+def get_long_lived_token(
+    access_token: str = dlt.secrets.value,
+    client_id: str = dlt.secrets.value,
+    client_secret: str = dlt.secrets.value,
+) -> str:
+    """Gets the long lived access token (60 days) from `access_token`. If arguments are not provides, `dlt` will inject them from configuration"""
+    exchange_url = f"https://graph.facebook.com/v13.0/oauth/access_token?grant_type=fb_exchange_token&client_id={client_id}&client_secret={client_secret}&fb_exchange_token={access_token}"
+    response = requests.get(exchange_url)
+    data: Dict[str, str] = response.json()
+
+    if "error" in data:
+        raise Exception(f"Error refreshing token: {data['error']}")
+
+    return data["access_token"]