ingestr 0.7.7__py3-none-any.whl → 0.8.1__py3-none-any.whl

ingestr/src/facebook_ads/helpers.py

@@ -0,0 +1,255 @@
+"""Facebook ads source helpers"""
+
+import functools
+import itertools
+import time
+from typing import Any, Iterator, Sequence
+
+import dlt
+import humanize
+import pendulum
+from dlt.common import logger
+from dlt.common.configuration.inject import with_config
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import DictStrAny, TDataItem, TDataItems
+from dlt.sources.helpers import requests
+from dlt.sources.helpers.requests import Client
+from facebook_business import FacebookAdsApi
+from facebook_business.adobjects.abstractcrudobject import AbstractCrudObject
+from facebook_business.adobjects.abstractobject import AbstractObject
+from facebook_business.adobjects.adaccount import AdAccount
+from facebook_business.adobjects.user import User
+from facebook_business.api import FacebookResponse
+
+from .exceptions import InsightsJobTimeout
+from .settings import (
+    FACEBOOK_INSIGHTS_RETENTION_PERIOD,
+    INSIGHTS_PRIMARY_KEY,
+    TFbMethod,
+)
+
+
+def get_start_date(
+    incremental_start_date: dlt.sources.incremental[str],
+    attribution_window_days_lag: int = 7,
+) -> pendulum.DateTime:
+    """
+    Get the start date for incremental loading of Facebook Insights data.
+    """
+    start_date: pendulum.DateTime = ensure_pendulum_datetime(
+        incremental_start_date.start_value
+    ).subtract(days=attribution_window_days_lag)
+
+    # facebook forgets insights so trim the lag and warn
+    min_start_date = pendulum.today().subtract(
+        months=FACEBOOK_INSIGHTS_RETENTION_PERIOD
+    )
+    if start_date < min_start_date:
+        logger.warning(
+            "%s: Start date is earlier than %s months ago, using %s instead. "
+            "For more information, see https://www.facebook.com/business/help/1695754927158071?id=354406972049255",
+            "facebook_insights",
+            FACEBOOK_INSIGHTS_RETENTION_PERIOD,
+            min_start_date,
+        )
+        start_date = min_start_date
+        incremental_start_date.start_value = min_start_date
+
+    # lag the incremental start date by attribution window lag
+    incremental_start_date.start_value = start_date.isoformat()
+    return start_date
+
+
+def process_report_item(item: AbstractObject) -> DictStrAny:
+    d: DictStrAny = item.export_all_data()
+    for pki in INSIGHTS_PRIMARY_KEY:
+        if pki not in d:
+            d[pki] = "no_" + pki
+
+    return d
+
+
+def get_data_chunked(
+    method: TFbMethod, fields: Sequence[str], states: Sequence[str], chunk_size: int
+) -> Iterator[TDataItems]:
+    # add pagination and chunk into lists
+    params: DictStrAny = {"limit": chunk_size}
+    if states:
+        params.update({"effective_status": states})
+    it: map[DictStrAny] = map(
+        lambda c: c.export_all_data(), method(fields=fields, params=params)
+    )
+    while True:
+        chunk = list(itertools.islice(it, chunk_size))
+        if not chunk:
+            break
+        yield chunk
+
+
+def enrich_ad_objects(fb_obj_type: AbstractObject, fields: Sequence[str]) -> Any:
+    """Returns a transformation that will enrich any of the resources returned by `` with additional fields
+
+    In example below we add "thumbnail_url" to all objects loaded by `ad_creatives` resource:
+    >>> fb_ads = facebook_ads_source()
+    >>> fb_ads.ad_creatives.add_step(enrich_ad_objects(AdCreative, ["thumbnail_url"]))
+
+    Internally, the method uses batch API to get data efficiently. Refer to demo script for full examples
+
+    Args:
+        fb_obj_type (AbstractObject): A Facebook Business object type (Ad, Campaign, AdSet, AdCreative, Lead). Import those types from this module
+        fields (Sequence[str]): A list/tuple of fields to add to each object.
+
+    Returns:
+        ItemTransformFunctionWithMeta[TDataItems]: A transformation function to be added to a resource with `add_step` method
+    """
+
+    def _wrap(items: TDataItems, meta: Any = None) -> TDataItems:
+        api_batch = FacebookAdsApi.get_default_api().new_batch()
+
+        def update_item(resp: FacebookResponse, item: TDataItem) -> None:
+            item.update(resp.json())
+
+        def fail(resp: FacebookResponse) -> None:
+            raise resp.error()
+
+        for item in items:
+            o: AbstractCrudObject = fb_obj_type(item["id"])
+            o.api_get(
+                fields=fields,
+                batch=api_batch,
+                success=functools.partial(update_item, item=item),
+                failure=fail,
+            )
+        api_batch.execute()
+        return items
+
+    return _wrap
+
+
+JOB_TIMEOUT_INFO = """This is an intermittent error and may resolve itself on subsequent queries to the Facebook API.
+You should remove the fields in `fields` argument that are not necessary, as that may help improve the reliability of the Facebook API."""
+
+
+def execute_job(
+    job: AbstractCrudObject,
+    insights_max_wait_to_start_seconds: int = 5 * 60,
+    insights_max_wait_to_finish_seconds: int = 30 * 60,
+    insights_max_async_sleep_seconds: int = 5 * 60,
+) -> AbstractCrudObject:
+    status: str = None
+    time_start = time.time()
+    sleep_time = 10
+    while status != "Job Completed":
+        duration = time.time() - time_start
+        job = job.api_get()
+        status = job["async_status"]
+        percent_complete = job["async_percent_completion"]
+
+        job_id = job["id"]
+        logger.info("%s, %d%% done", status, percent_complete)
+
+        if status == "Job Completed":
+            return job
+
+        if duration > insights_max_wait_to_start_seconds and percent_complete == 0:
+            pretty_error_message = (
+                "Insights job {} did not start after {} seconds. " + JOB_TIMEOUT_INFO
+            )
+            raise InsightsJobTimeout(
+                "facebook_insights",
+                pretty_error_message.format(job_id, insights_max_wait_to_start_seconds),
+            )
+        elif (
+            duration > insights_max_wait_to_finish_seconds and status != "Job Completed"
+        ):
+            pretty_error_message = (
+                "Insights job {} did not complete after {} seconds. " + JOB_TIMEOUT_INFO
+            )
+            raise InsightsJobTimeout(
+                "facebook_insights",
+                pretty_error_message.format(
+                    job_id, insights_max_wait_to_finish_seconds // 60
+                ),
+            )
+
+        logger.info("sleeping for %d seconds until job is done", sleep_time)
+        time.sleep(sleep_time)
+        if sleep_time < insights_max_async_sleep_seconds:
+            sleep_time = 2 * sleep_time
+    return job
+
+
+def get_ads_account(
+    account_id: str, access_token: str, request_timeout: float, app_api_version: str
+) -> AdAccount:
+    notify_on_token_expiration()
+
+    def retry_on_limit(response: requests.Response, exception: BaseException) -> bool:
+        try:
+            error = response.json()["error"]
+            code = error["code"]
+            message = error["message"]
+            should_retry = code in (
+                1,
+                2,
+                4,
+                17,
+                341,
+                32,
+                613,
+                *range(80000, 80007),
+                800008,
+                800009,
+                80014,
+            )
+            if should_retry:
+                logger.warning(
+                    "facebook_ads source will retry due to %s with error code %i"
+                    % (message, code)
+                )
+            return should_retry
+        except Exception:
+            return False
+
+    retry_session = Client(
+        request_timeout=request_timeout,
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+        request_backoff_factor=2,
+    ).session
+    retry_session.params.update({"access_token": access_token})  # type: ignore
+    # patch dlt requests session with retries
+    API = FacebookAdsApi.init(
+        account_id="act_" + account_id,
+        access_token=access_token,
+        api_version=app_api_version,
+    )
+    API._session.requests = retry_session
+    user = User(fbid="me")
+
+    accounts = user.get_ad_accounts()
+    account: AdAccount = None
+    for acc in accounts:
+        if acc["account_id"] == account_id:
+            account = acc
+
+    if not account:
+        raise ValueError("Couldn't find account with id {}".format(account_id))
+
+    return account
+
+
+@with_config(sections=("sources", "facebook_ads"))
+def notify_on_token_expiration(access_token_expires_at: int = None) -> None:
+    """Notifies (currently via logger) if access token expires in less than 7 days. Needs `access_token_expires_at` to be configured."""
+    if not access_token_expires_at:
+        logger.warning(
+            "Token expiration time notification disabled. Configure token expiration timestamp in access_token_expires_at config value"
+        )
+    else:
+        expires_at = pendulum.from_timestamp(access_token_expires_at)
+        if expires_at < pendulum.now().add(days=7):
+            logger.error(
+                f"Access Token expires in {humanize.precisedelta(pendulum.now() - expires_at)}. Replace the token now!"
+            )

ingestr/src/facebook_ads/settings.py

@@ -0,0 +1,208 @@
+"""Facebook ads source settings and constants"""
+
+from typing import Any, Callable, Dict, Iterator, Literal
+
+from dlt.common.schema.typing import TTableSchemaColumns
+from facebook_business.adobjects.abstractobject import AbstractObject
+
+TFbMethod = Callable[..., Iterator[AbstractObject]]
+
+
+DEFAULT_FIELDS = (
+    "id",
+    "updated_time",
+    "created_time",
+    "name",
+    "status",
+    "effective_status",
+)
+
+DEFAULT_CAMPAIGN_FIELDS = DEFAULT_FIELDS + (
+    "objective",
+    "start_time",
+    "stop_time",
+    "daily_budget",
+    "lifetime_budget",
+)
+
+DEFAULT_AD_FIELDS = DEFAULT_FIELDS + (
+    "adset_id",
+    "campaign_id",
+    "creative",
+    "targeting",
+    "tracking_specs",
+    "conversion_specs",
+)
+
+DEFAULT_ADSET_FIELDS = DEFAULT_FIELDS + (
+    "campaign_id",
+    "start_time",
+    "end_time",
+    "daily_budget",
+    "lifetime_budget",
+    "optimization_goal",
+    "promoted_object",
+    "billing_event",
+    "bid_amount",
+    "bid_strategy",
+    "targeting",
+)
+
+DEFAULT_ADCREATIVE_FIELDS = (
+    "id",
+    "name",
+    "status",
+    "thumbnail_url",
+    "object_story_spec",
+    "effective_object_story_id",
+    "call_to_action_type",
+    "object_type",
+    "template_url",
+    "url_tags",
+    "instagram_actor_id",
+    "product_set_id",
+)
+
+DEFAULT_LEAD_FIELDS = (
+    "id",
+    "created_time",
+    "ad_id",
+    "ad_name",
+    "adset_id",
+    "adset_name",
+    "campaign_id",
+    "campaign_name",
+    "form_id",
+    "field_data",
+)
+
+DEFAULT_INSIGHT_FIELDS = (
+    "campaign_id",
+    "adset_id",
+    "ad_id",
+    "date_start",
+    "date_stop",
+    "reach",
+    "impressions",
+    "frequency",
+    "clicks",
+    "unique_clicks",
+    "ctr",
+    "unique_ctr",
+    "cpc",
+    "cpm",
+    "cpp",
+    "spend",
+    "actions",
+    "action_values",
+    "cost_per_action_type",
+    "website_ctr",
+    "account_currency",
+    "ad_click_actions",
+    "ad_name",
+    "adset_name",
+    "campaign_name",
+    "country",
+    "dma",
+    "full_view_impressions",
+    "full_view_reach",
+    "inline_link_click_ctr",
+    "outbound_clicks",
+    "reach",
+    "social_spend",
+    "spend",
+    "website_ctr",
+)
+
+TInsightsLevels = Literal["account", "campaign", "adset", "ad"]
+
+INSIGHTS_PRIMARY_KEY = ("campaign_id", "adset_id", "ad_id", "date_start")
+
+ALL_STATES = {
+    "effective_status": [
+        "ACTIVE",
+        "PAUSED",
+        "DELETED",
+        "PENDING_REVIEW",
+        "DISAPPROVED",
+        "PREAPPROVED",
+        "PENDING_BILLING_INFO",
+        "CAMPAIGN_PAUSED",
+        "ARCHIVED",
+        "ADSET_PAUSED",
+    ]
+}
+
+TInsightsBreakdownOptions = Literal[
+    "ads_insights",
+    "ads_insights_age_and_gender",
+    "ads_insights_country",
+    "ads_insights_platform_and_device",
+    "ads_insights_region",
+    "ads_insights_dma",
+    "ads_insights_hourly_advertiser",
+]
+
+ALL_ACTION_ATTRIBUTION_WINDOWS = (
+    "1d_click",
+    "7d_click",
+    "28d_click",
+    "1d_view",
+    "7d_view",
+    "28d_view",
+)
+
+ALL_ACTION_BREAKDOWNS = ("action_type", "action_target_id", "action_destination")
+
+INSIGHTS_BREAKDOWNS_OPTIONS: Dict[TInsightsBreakdownOptions, Any] = {
+    "ads_insights": {"breakdowns": (), "fields": ()},
+    "ads_insights_age_and_gender": {
+        "breakdowns": ("age", "gender"),
+        "fields": ("age", "gender"),
+    },
+    "ads_insights_country": {"breakdowns": ("country",), "fields": ("country",)},
+    "ads_insights_platform_and_device": {
+        "breakdowns": ("publisher_platform", "platform_position", "impression_device"),
+        "fields": ("publisher_platform", "platform_position", "impression_device"),
+    },
+    "ads_insights_region": {"breakdowns": ("region",), "fields": ("region",)},
+    "ads_insights_dma": {"breakdowns": ("dma",), "fields": ("dma",)},
+    "ads_insights_hourly_advertiser": {
+        "breakdowns": ("hourly_stats_aggregated_by_advertiser_time_zone",),
+        "fields": ("hourly_stats_aggregated_by_advertiser_time_zone",),
+    },
+}
+
+INSIGHT_FIELDS_TYPES: TTableSchemaColumns = {
+    "campaign_id": {"data_type": "bigint"},
+    "adset_id": {"data_type": "bigint"},
+    "ad_id": {"data_type": "bigint"},
+    "date_start": {"data_type": "timestamp"},
+    "date_stop": {"data_type": "timestamp"},
+    "reach": {"data_type": "bigint"},
+    "impressions": {"data_type": "bigint"},
+    "frequency": {"data_type": "decimal"},
+    "clicks": {"data_type": "bigint"},
+    "unique_clicks": {"data_type": "bigint"},
+    "ctr": {"data_type": "decimal"},
+    "unique_ctr": {"data_type": "decimal"},
+    "cpc": {"data_type": "decimal"},
+    "cpm": {"data_type": "decimal"},
+    "cpp": {"data_type": "decimal"},
+    "spend": {"data_type": "decimal"},
+}
+
+INVALID_INSIGHTS_FIELDS = [
+    "impression_device",
+    "publisher_platform",
+    "platform_position",
+    "age",
+    "gender",
+    "country",
+    "placement",
+    "region",
+    "dma",
+    "hourly_stats_aggregated_by_advertiser_time_zone",
+]
+
+FACEBOOK_INSIGHTS_RETENTION_PERIOD = 37  # months

ingestr/src/factory.py

@@ -15,11 +15,16 @@ from ingestr.src.destinations import (
     SynapseDestination,
 )
 from ingestr.src.sources import (
+    AdjustSource,
     AirtableSource,
+    AppsflyerSource,
     ChessSource,
+    FacebookAdsSource,
     GoogleSheetsSource,
     GorgiasSource,
     HubspotSource,
+    KafkaSource,
+    KlaviyoSource,
     LocalCsvSource,
     MongoDbSource,
     NotionSource,
@@ -111,12 +116,22 @@ class SourceDestinationFactory:
             return ChessSource()
         elif self.source_scheme == "stripe":
             return StripeAnalyticsSource()
+        elif self.source_scheme == "facebookads":
+            return FacebookAdsSource()
         elif self.source_scheme == "slack":
             return SlackSource()
         elif self.source_scheme == "hubspot":
             return HubspotSource()
         elif self.source_scheme == "airtable":
             return AirtableSource()
+        elif self.source_scheme == "klaviyo":
+            return KlaviyoSource()
+        elif self.source_scheme == "appsflyer":
+            return AppsflyerSource()
+        elif self.source_scheme == "kafka":
+            return KafkaSource()
+        elif self.source_scheme == "adjust":
+            return AdjustSource()
         else:
             raise ValueError(f"Unsupported source scheme: {self.source_scheme}")
 

ingestr/src/kafka/__init__.py

@@ -0,0 +1,103 @@
+"""A source to extract Kafka messages.
+
+When extraction starts, partitions length is checked -
+data is read only up to it, overriding the default Kafka's
+behavior of waiting for new messages in endless loop.
+"""
+
+from contextlib import closing
+from typing import Any, Callable, Dict, Iterable, List, Optional, Union
+
+import dlt
+from confluent_kafka import Consumer, Message  # type: ignore
+from dlt.common import logger
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime, TDataItem
+
+from .helpers import (
+    KafkaCredentials,
+    OffsetTracker,
+    default_msg_processor,
+)
+
+
+@dlt.resource(
+    name="kafka_messages",
+    table_name=lambda msg: msg["_kafka"]["topic"],
+    standalone=True,
+)
+def kafka_consumer(
+    topics: Union[str, List[str]],
+    credentials: Union[KafkaCredentials, Consumer] = dlt.secrets.value,
+    msg_processor: Optional[
+        Callable[[Message], Dict[str, Any]]
+    ] = default_msg_processor,
+    batch_size: Optional[int] = 3000,
+    batch_timeout: Optional[int] = 3,
+    start_from: Optional[TAnyDateTime] = None,
+) -> Iterable[TDataItem]:
+    """Extract recent messages from the given Kafka topics.
+
+    The resource tracks offsets for all the topics and partitions,
+    and so reads data incrementally.
+
+    Messages from different topics are saved in different tables.
+
+    Args:
+        topics (Union[str, List[str]]): Names of topics to extract.
+        credentials (Optional[Union[KafkaCredentials, Consumer]]):
+            Auth credentials or an initiated Kafka consumer. By default,
+            is taken from secrets.
+        msg_processor(Optional[Callable]): A function-converter,
+            which'll process every Kafka message after it's read and
+            before it's transfered to the destination.
+        batch_size (Optional[int]): Messages batch size to read at once.
+        batch_timeout (Optional[int]): Maximum time to wait for a batch
+            consume, in seconds.
+        start_from (Optional[TAnyDateTime]): A timestamp, at which to start
+            reading. Older messages are ignored.
+
+    Yields:
+        Iterable[TDataItem]: Kafka messages.
+    """
+    if not isinstance(topics, list):
+        topics = [topics]
+
+    if isinstance(credentials, Consumer):
+        consumer = credentials
+    elif isinstance(credentials, KafkaCredentials):
+        consumer = credentials.init_consumer()
+    else:
+        raise TypeError(
+            (
+                "Wrong credentials type provided. Need to be of type: "
+                "KafkaCredentials or confluent_kafka.Consumer"
+            )
+        )
+
+    if start_from is not None:
+        start_from = ensure_pendulum_datetime(start_from)
+
+    tracker = OffsetTracker(consumer, topics, dlt.current.resource_state(), start_from)  # type: ignore
+
+    # read messages up to the maximum offsets,
+    # not waiting for new messages
+    with closing(consumer):
+        while tracker.has_unread:
+            messages = consumer.consume(batch_size, timeout=batch_timeout)
+            if not messages:
+                break
+
+            batch = []
+            for msg in messages:
+                if msg.error():
+                    err = msg.error()
+                    if err.retriable() or not err.fatal():
+                        logger.warning(f"ERROR: {err} - RETRYING")
+                    else:
+                        raise err
+                else:
+                    batch.append(msg_processor(msg))  # type: ignore
+                    tracker.renew(msg)
+
+            yield batch