ingestr 0.7.6__py3-none-any.whl → 0.7.8__py3-none-any.whl

ingestr/src/facebook_ads/settings.py

@@ -0,0 +1,208 @@
+"""Facebook ads source settings and constants"""
+
+from typing import Any, Callable, Dict, Iterator, Literal
+
+from dlt.common.schema.typing import TTableSchemaColumns
+from facebook_business.adobjects.abstractobject import AbstractObject
+
+TFbMethod = Callable[..., Iterator[AbstractObject]]
+
+
+DEFAULT_FIELDS = (
+    "id",
+    "updated_time",
+    "created_time",
+    "name",
+    "status",
+    "effective_status",
+)
+
+DEFAULT_CAMPAIGN_FIELDS = DEFAULT_FIELDS + (
+    "objective",
+    "start_time",
+    "stop_time",
+    "daily_budget",
+    "lifetime_budget",
+)
+
+DEFAULT_AD_FIELDS = DEFAULT_FIELDS + (
+    "adset_id",
+    "campaign_id",
+    "creative",
+    "targeting",
+    "tracking_specs",
+    "conversion_specs",
+)
+
+DEFAULT_ADSET_FIELDS = DEFAULT_FIELDS + (
+    "campaign_id",
+    "start_time",
+    "end_time",
+    "daily_budget",
+    "lifetime_budget",
+    "optimization_goal",
+    "promoted_object",
+    "billing_event",
+    "bid_amount",
+    "bid_strategy",
+    "targeting",
+)
+
+DEFAULT_ADCREATIVE_FIELDS = (
+    "id",
+    "name",
+    "status",
+    "thumbnail_url",
+    "object_story_spec",
+    "effective_object_story_id",
+    "call_to_action_type",
+    "object_type",
+    "template_url",
+    "url_tags",
+    "instagram_actor_id",
+    "product_set_id",
+)
+
+DEFAULT_LEAD_FIELDS = (
+    "id",
+    "created_time",
+    "ad_id",
+    "ad_name",
+    "adset_id",
+    "adset_name",
+    "campaign_id",
+    "campaign_name",
+    "form_id",
+    "field_data",
+)
+
+DEFAULT_INSIGHT_FIELDS = (
+    "campaign_id",
+    "adset_id",
+    "ad_id",
+    "date_start",
+    "date_stop",
+    "reach",
+    "impressions",
+    "frequency",
+    "clicks",
+    "unique_clicks",
+    "ctr",
+    "unique_ctr",
+    "cpc",
+    "cpm",
+    "cpp",
+    "spend",
+    "actions",
+    "action_values",
+    "cost_per_action_type",
+    "website_ctr",
+    "account_currency",
+    "ad_click_actions",
+    "ad_name",
+    "adset_name",
+    "campaign_name",
+    "country",
+    "dma",
+    "full_view_impressions",
+    "full_view_reach",
+    "inline_link_click_ctr",
+    "outbound_clicks",
+    "reach",
+    "social_spend",
+    "spend",
+    "website_ctr",
+)
+
+TInsightsLevels = Literal["account", "campaign", "adset", "ad"]
+
+INSIGHTS_PRIMARY_KEY = ("campaign_id", "adset_id", "ad_id", "date_start")
+
+ALL_STATES = {
+    "effective_status": [
+        "ACTIVE",
+        "PAUSED",
+        "DELETED",
+        "PENDING_REVIEW",
+        "DISAPPROVED",
+        "PREAPPROVED",
+        "PENDING_BILLING_INFO",
+        "CAMPAIGN_PAUSED",
+        "ARCHIVED",
+        "ADSET_PAUSED",
+    ]
+}
+
+TInsightsBreakdownOptions = Literal[
+    "ads_insights",
+    "ads_insights_age_and_gender",
+    "ads_insights_country",
+    "ads_insights_platform_and_device",
+    "ads_insights_region",
+    "ads_insights_dma",
+    "ads_insights_hourly_advertiser",
+]
+
+ALL_ACTION_ATTRIBUTION_WINDOWS = (
+    "1d_click",
+    "7d_click",
+    "28d_click",
+    "1d_view",
+    "7d_view",
+    "28d_view",
+)
+
+ALL_ACTION_BREAKDOWNS = ("action_type", "action_target_id", "action_destination")
+
+INSIGHTS_BREAKDOWNS_OPTIONS: Dict[TInsightsBreakdownOptions, Any] = {
+    "ads_insights": {"breakdowns": (), "fields": ()},
+    "ads_insights_age_and_gender": {
+        "breakdowns": ("age", "gender"),
+        "fields": ("age", "gender"),
+    },
+    "ads_insights_country": {"breakdowns": ("country",), "fields": ("country",)},
+    "ads_insights_platform_and_device": {
+        "breakdowns": ("publisher_platform", "platform_position", "impression_device"),
+        "fields": ("publisher_platform", "platform_position", "impression_device"),
+    },
+    "ads_insights_region": {"breakdowns": ("region",), "fields": ("region",)},
+    "ads_insights_dma": {"breakdowns": ("dma",), "fields": ("dma",)},
+    "ads_insights_hourly_advertiser": {
+        "breakdowns": ("hourly_stats_aggregated_by_advertiser_time_zone",),
+        "fields": ("hourly_stats_aggregated_by_advertiser_time_zone",),
+    },
+}
+
+INSIGHT_FIELDS_TYPES: TTableSchemaColumns = {
+    "campaign_id": {"data_type": "bigint"},
+    "adset_id": {"data_type": "bigint"},
+    "ad_id": {"data_type": "bigint"},
+    "date_start": {"data_type": "timestamp"},
+    "date_stop": {"data_type": "timestamp"},
+    "reach": {"data_type": "bigint"},
+    "impressions": {"data_type": "bigint"},
+    "frequency": {"data_type": "decimal"},
+    "clicks": {"data_type": "bigint"},
+    "unique_clicks": {"data_type": "bigint"},
+    "ctr": {"data_type": "decimal"},
+    "unique_ctr": {"data_type": "decimal"},
+    "cpc": {"data_type": "decimal"},
+    "cpm": {"data_type": "decimal"},
+    "cpp": {"data_type": "decimal"},
+    "spend": {"data_type": "decimal"},
+}
+
+INVALID_INSIGHTS_FIELDS = [
+    "impression_device",
+    "publisher_platform",
+    "platform_position",
+    "age",
+    "gender",
+    "country",
+    "placement",
+    "region",
+    "dma",
+    "hourly_stats_aggregated_by_advertiser_time_zone",
+]
+
+FACEBOOK_INSIGHTS_RETENTION_PERIOD = 37  # months

ingestr/src/factory.py

@@ -15,14 +15,19 @@ from ingestr.src.destinations import (
     SynapseDestination,
 )
 from ingestr.src.sources import (
+    AirtableSource,
     ChessSource,
+    FacebookAdsSource,
     GoogleSheetsSource,
     GorgiasSource,
     HubspotSource,
+    KafkaSource,
+    KlaviyoSource,
     LocalCsvSource,
     MongoDbSource,
     NotionSource,
     ShopifySource,
+    SlackSource,
     SqlSource,
     StripeAnalyticsSource,
 )
@@ -109,8 +114,18 @@ class SourceDestinationFactory:
             return ChessSource()
         elif self.source_scheme == "stripe":
             return StripeAnalyticsSource()
+        elif self.source_scheme == "facebookads":
+            return FacebookAdsSource()
+        elif self.source_scheme == "slack":
+            return SlackSource()
         elif self.source_scheme == "hubspot":
             return HubspotSource()
+        elif self.source_scheme == "airtable":
+            return AirtableSource()
+        elif self.source_scheme == "klaviyo":
+            return KlaviyoSource()
+        elif self.source_scheme == "kafka":
+            return KafkaSource()
         else:
             raise ValueError(f"Unsupported source scheme: {self.source_scheme}")
 

ingestr/src/kafka/__init__.py

@@ -0,0 +1,103 @@
+"""A source to extract Kafka messages.
+
+When extraction starts, partitions length is checked -
+data is read only up to it, overriding the default Kafka's
+behavior of waiting for new messages in endless loop.
+"""
+
+from contextlib import closing
+from typing import Any, Callable, Dict, Iterable, List, Optional, Union
+
+import dlt
+from confluent_kafka import Consumer, Message  # type: ignore
+from dlt.common import logger
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime, TDataItem
+
+from .helpers import (
+    KafkaCredentials,
+    OffsetTracker,
+    default_msg_processor,
+)
+
+
+@dlt.resource(
+    name="kafka_messages",
+    table_name=lambda msg: msg["_kafka"]["topic"],
+    standalone=True,
+)
+def kafka_consumer(
+    topics: Union[str, List[str]],
+    credentials: Union[KafkaCredentials, Consumer] = dlt.secrets.value,
+    msg_processor: Optional[
+        Callable[[Message], Dict[str, Any]]
+    ] = default_msg_processor,
+    batch_size: Optional[int] = 3000,
+    batch_timeout: Optional[int] = 3,
+    start_from: Optional[TAnyDateTime] = None,
+) -> Iterable[TDataItem]:
+    """Extract recent messages from the given Kafka topics.
+
+    The resource tracks offsets for all the topics and partitions,
+    and so reads data incrementally.
+
+    Messages from different topics are saved in different tables.
+
+    Args:
+        topics (Union[str, List[str]]): Names of topics to extract.
+        credentials (Optional[Union[KafkaCredentials, Consumer]]):
+            Auth credentials or an initiated Kafka consumer. By default,
+            is taken from secrets.
+        msg_processor(Optional[Callable]): A function-converter,
+            which'll process every Kafka message after it's read and
+            before it's transfered to the destination.
+        batch_size (Optional[int]): Messages batch size to read at once.
+        batch_timeout (Optional[int]): Maximum time to wait for a batch
+            consume, in seconds.
+        start_from (Optional[TAnyDateTime]): A timestamp, at which to start
+            reading. Older messages are ignored.
+
+    Yields:
+        Iterable[TDataItem]: Kafka messages.
+    """
+    if not isinstance(topics, list):
+        topics = [topics]
+
+    if isinstance(credentials, Consumer):
+        consumer = credentials
+    elif isinstance(credentials, KafkaCredentials):
+        consumer = credentials.init_consumer()
+    else:
+        raise TypeError(
+            (
+                "Wrong credentials type provided. Need to be of type: "
+                "KafkaCredentials or confluent_kafka.Consumer"
+            )
+        )
+
+    if start_from is not None:
+        start_from = ensure_pendulum_datetime(start_from)
+
+    tracker = OffsetTracker(consumer, topics, dlt.current.resource_state(), start_from)  # type: ignore
+
+    # read messages up to the maximum offsets,
+    # not waiting for new messages
+    with closing(consumer):
+        while tracker.has_unread:
+            messages = consumer.consume(batch_size, timeout=batch_timeout)
+            if not messages:
+                break
+
+            batch = []
+            for msg in messages:
+                if msg.error():
+                    err = msg.error()
+                    if err.retriable() or not err.fatal():
+                        logger.warning(f"ERROR: {err} - RETRYING")
+                    else:
+                        raise err
+                else:
+                    batch.append(msg_processor(msg))  # type: ignore
+                    tracker.renew(msg)
+
+            yield batch

ingestr/src/kafka/helpers.py

@@ -0,0 +1,227 @@
+from typing import Any, Dict, List, Optional
+
+from confluent_kafka import Consumer, Message, TopicPartition  # type: ignore
+from confluent_kafka.admin import TopicMetadata  # type: ignore
+from dlt import config
+from dlt.common import pendulum
+from dlt.common.configuration import configspec
+from dlt.common.configuration.specs import CredentialsConfiguration
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import DictStrAny, TSecretValue
+from dlt.common.utils import digest128
+
+
+def default_msg_processor(msg: Message) -> Dict[str, Any]:
+    """Basic Kafka message processor.
+
+    Returns the message value and metadata. Timestamp consists of two values:
+    (type of the timestamp, timestamp). Type represents one of the Python
+    Kafka constants:
+        TIMESTAMP_NOT_AVAILABLE - Timestamps not supported by broker.
+        TIMESTAMP_CREATE_TIME - Message creation time (or source / producer time).
+        TIMESTAMP_LOG_APPEND_TIME - Broker receive time.
+
+    Args:
+        msg (confluent_kafka.Message): A single Kafka message.
+
+    Returns:
+        dict: Processed Kafka message.
+    """
+    ts = msg.timestamp()
+    topic = msg.topic()
+    partition = msg.partition()
+    key = msg.key()
+    if key is not None:
+        key = key.decode("utf-8")
+
+    return {
+        "_kafka": {
+            "partition": partition,
+            "topic": topic,
+            "key": key,
+            "offset": msg.offset(),
+            "ts": {
+                "type": ts[0],
+                "value": ensure_pendulum_datetime(ts[1] / 1e3),
+            },
+            "data": msg.value().decode("utf-8"),
+        },
+        "_kafka_msg_id": digest128(topic + str(partition) + str(key)),
+    }
+
+
+class OffsetTracker(dict):  # type: ignore
+    """Object to control offsets of the given topics.
+
+    Tracks all the partitions of the given topics with two params:
+    current offset and maximum offset (partition length).
+
+    Args:
+        consumer (confluent_kafka.Consumer): Kafka consumer.
+        topic_names (List): Names of topics to track.
+        pl_state (DictStrAny): Pipeline current state.
+        start_from (Optional[pendulum.DateTime]): A timestamp, after which messages
+            are read. Older messages are ignored.
+    """
+
+    def __init__(
+        self,
+        consumer: Consumer,
+        topic_names: List[str],
+        pl_state: DictStrAny,
+        start_from: pendulum.DateTime = None,  # type: ignore
+    ):
+        super().__init__()
+
+        self._consumer = consumer
+        self._topics = self._read_topics(topic_names)
+
+        # read/init current offsets
+        self._cur_offsets = pl_state.setdefault(
+            "offsets", {t_name: {} for t_name in topic_names}
+        )
+
+        self._init_partition_offsets(start_from)
+
+    def _read_topics(self, topic_names: List[str]) -> Dict[str, TopicMetadata]:
+        """Read the given topics metadata from Kafka.
+
+        Reads all the topics at once, instead of requesting
+        each in a separate call. Returns only those needed.
+
+        Args:
+            topic_names (list): Names of topics to be read.
+
+        Returns:
+            dict: Metadata of the given topics.
+        """
+        tracked_topics = {}
+        topics = self._consumer.list_topics().topics
+
+        for t_name in topic_names:
+            tracked_topics[t_name] = topics[t_name]
+
+        return tracked_topics
+
+    def _init_partition_offsets(self, start_from: pendulum.DateTime) -> None:
+        """Designate current and maximum offsets for every partition.
+
+        Current offsets are read from the state, if present. Set equal
+        to the partition beginning otherwise.
+
+        Args:
+            start_from (pendulum.DateTime): A timestamp, at which to start
+                reading. Older messages are ignored.
+        """
+        all_parts = []
+        for t_name, topic in self._topics.items():
+            self[t_name] = {}
+
+            # init all the topic partitions from the partitions' metadata
+            parts = [
+                TopicPartition(
+                    t_name,
+                    part,
+                    start_from.int_timestamp * 1000 if start_from is not None else 0,
+                )
+                for part in topic.partitions
+            ]
+
+            # get offsets for the timestamp, if given
+            if start_from is not None:
+                ts_offsets = self._consumer.offsets_for_times(parts)
+
+            # designate current and maximum offsets for every partition
+            for i, part in enumerate(parts):
+                max_offset = self._consumer.get_watermark_offsets(part)[1]
+
+                if start_from is not None:
+                    if ts_offsets[i].offset != -1:
+                        cur_offset = ts_offsets[i].offset
+                    else:
+                        cur_offset = max_offset - 1
+                else:
+                    cur_offset = (
+                        self._cur_offsets[t_name].get(str(part.partition), -1) + 1
+                    )
+
+                self[t_name][str(part.partition)] = {
+                    "cur": cur_offset,
+                    "max": max_offset,
+                }
+
+                parts[i].offset = cur_offset
+
+            all_parts += parts
+
+        # assign the current offsets to the consumer
+        self._consumer.assign(all_parts)
+
+    @property
+    def has_unread(self) -> bool:
+        """Check if there are unread messages in the tracked topics.
+
+        Returns:
+            bool: True, if there are messages to read, False if all
+                the current offsets are equal to their maximums.
+        """
+        for parts in self.values():
+            for part in parts.values():
+                if part["cur"] + 1 < part["max"]:
+                    return True
+
+        return False
+
+    def renew(self, msg: Message) -> None:
+        """Update partition offset from the given message.
+
+        Args:
+            msg (confluent_kafka.Message): A read Kafka message.
+        """
+        topic = msg.topic()
+        partition = str(msg.partition())
+
+        offset = self[topic][partition]
+        offset["cur"] = msg.offset()
+
+        self._cur_offsets[topic][partition] = msg.offset()
+
+
+@configspec
+class KafkaCredentials(CredentialsConfiguration):
+    """Kafka source credentials.
+
+    NOTE: original Kafka credentials are written with a period, e.g.
+    bootstrap.servers. However, KafkaCredentials expect them to
+    use underscore symbols instead, e.g. bootstrap_servers.
+    """
+
+    bootstrap_servers: str = config.value
+    group_id: str = config.value
+    security_protocol: Optional[str] = None
+    sasl_mechanisms: Optional[str] = None
+    sasl_username: Optional[str] = None
+    sasl_password: Optional[TSecretValue] = None
+
+    def init_consumer(self) -> Consumer:
+        """Init a Kafka consumer from this credentials.
+
+        Returns:
+            confluent_kafka.Consumer: an initiated consumer.
+        """
+        config = {
+            "bootstrap.servers": self.bootstrap_servers,
+            "group.id": self.group_id,
+            "auto.offset.reset": "earliest",
+        }
+
+        if self.security_protocol:
+            config["security.protocol"] = self.security_protocol
+        if self.sasl_mechanisms:
+            config["sasl.mechanisms"] = self.sasl_mechanisms
+        if self.sasl_username:
+            config["sasl.username"] = self.sasl_username
+        if self.sasl_password:
+            config["sasl.password"] = self.sasl_password
+
+        return Consumer(config)