ingestr 0.7.6__py3-none-any.whl → 0.7.8__py3-none-any.whl

ingestr/src/klaviyo/_init_.py

@@ -0,0 +1,173 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+import requests
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers.requests import Client
+
+from ingestr.src.klaviyo.client import KlaviyoClient
+from ingestr.src.klaviyo.helpers import split_date_range
+
+
+def retry_on_limit(response: requests.Response, exception: BaseException) -> bool:
+    return response.status_code == 429
+
+
+def create_client() -> requests.Session:
+    return Client(
+        request_timeout=10.0,
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+        request_backoff_factor=2,
+    ).session
+
+
+@dlt.source(max_table_nesting=0)
+def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResource]:
+    start_date_obj = ensure_pendulum_datetime(start_date)
+    client = KlaviyoClient(api_key)
+
+    @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
+    def events(
+        datetime=dlt.sources.incremental("datetime", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(datetime.start_value), pendulum.now()
+        )
+
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_events(create_client(), s, e)
+
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
+    def profiles(
+        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated.start_value), pendulum.now()
+        )
+
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_profiles(create_client(), s, e)
+
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
+    def campaigns(
+        updated_at=dlt.sources.incremental("updated_at", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated_at.start_value), pendulum.now()
+        )
+
+        for campaign_type in ["email", "sms"]:
+            for start, end in intervals:
+                yield lambda s=start, e=end, ct=campaign_type: client.fetch_campaigns(
+                    create_client(), s, e, ct
+                )
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def metrics(
+        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_metrics(create_client(), updated.start_value)
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def tags() -> Iterable[TAnyDateTime]:
+        yield from client.fetch_tag(create_client())
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def coupons() -> Iterable[TAnyDateTime]:
+        yield from client.fetch_coupons(create_client())
+
+    @dlt.resource(write_disposition="merge", primary_key="id", name="catalog-variants")
+    def catalog_variants(
+        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_catalog_variant(create_client(), updated.start_value)
+
+    @dlt.resource(
+        write_disposition="merge", primary_key="id", name="catalog-categories"
+    )
+    def catalog_categories(
+        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_catalog_categories(create_client(), updated.start_value)
+
+    @dlt.resource(write_disposition="merge", primary_key="id", name="catalog-items")
+    def catalog_items(
+        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_catalog_item(create_client(), updated.start_value)
+
+    @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
+    def forms(
+        updated_at=dlt.sources.incremental("updated_at", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated_at.start_value), pendulum.now()
+        )
+
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_forms(create_client(), s, e)
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def lists(
+        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_lists(create_client(), updated.start_value)
+
+    @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
+    def images(
+        updated_at=dlt.sources.incremental("updated_at", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated_at.start_value), pendulum.now()
+        )
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_images(create_client(), s, e)
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def segments(
+        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_segments(create_client(), updated.start_value)
+
+    @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
+    def flows(
+        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated.start_value), pendulum.now()
+        )
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_flows(create_client(), s, e)
+
+    @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
+    def templates(
+        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated.start_value), pendulum.now()
+        )
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_templates(create_client(), s, e)
+
+    return (
+        events,
+        profiles,
+        campaigns,
+        metrics,
+        tags,
+        coupons,
+        catalog_variants,
+        catalog_categories,
+        catalog_items,
+        forms,
+        lists,
+        images,
+        segments,
+        flows,
+        templates,
+    )

ingestr/src/klaviyo/client.py

@@ -0,0 +1,212 @@
+from urllib.parse import urlencode
+
+import pendulum
+import requests
+
+BASE_URL = "https://a.klaviyo.com/api"
+
+
+class KlaviyoClient:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+
+    def __get_headers(self):
+        return {
+            "Authorization": f"Klaviyo-API-Key {self.api_key}",
+            "accept": "application/json",
+            "revision": "2024-07-15",
+        }
+
+    def _flatten_attributes(self, items: list):
+        for event in items:
+            if "attributes" not in event:
+                continue
+
+            for attribute_key in event["attributes"]:
+                event[attribute_key] = event["attributes"][attribute_key]
+
+            del event["attributes"]
+        return items
+
+    def _fetch_pages(
+        self, session: requests.Session, url: str, flat: bool = True
+    ) -> list:
+        all_items = []
+        while True:
+            response = session.get(url=url, headers=self.__get_headers())
+            result = response.json()
+            items = result.get("data", [])
+
+            if flat:
+                items = self._flatten_attributes(items)
+
+            all_items.extend(items)
+            nextURL = result.get("links", {}).get("next")
+            if nextURL is None:
+                break
+
+            url = nextURL
+
+        return all_items
+
+    def fetch_events(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+    ):
+        print(f"Fetching events for {start_date} to {end_date}")
+        url = f"{BASE_URL}/events/?sort=-datetime&filter=and(greater-or-equal(datetime,{start_date}),less-than(datetime,{end_date}))"
+        return self._fetch_pages(session, url)
+
+    def fetch_metrics(
+        self,
+        session: requests.Session,
+        last_updated: str,
+    ):
+        print(f"Fetching metrics since {last_updated}")
+        url = f"{BASE_URL}/metrics"
+        items = self._fetch_pages(session, url)
+
+        last_updated_obj = pendulum.parse(last_updated)
+        for item in items:
+            updated_at = pendulum.parse(item["updated"])
+            if updated_at > last_updated_obj:
+                yield item
+
+    def fetch_profiles(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+    ):
+        pendulum_start_date = pendulum.parse(start_date)
+        pendulum_start_date = pendulum_start_date.subtract(seconds=1)
+        url = f"{BASE_URL}/profiles/?sort=updated&filter=and(greater-than(updated,{pendulum_start_date.isoformat()}),less-than(updated,{end_date}))"
+        return self._fetch_pages(session, url)
+
+    def fetch_campaigns(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+        campaign_type: str,
+    ):
+        print(f"Fetching {campaign_type} campaigns for {start_date} to {end_date}")
+
+        base_url = f"{BASE_URL}/campaigns/"
+        params = {
+            "sort": "updated_at",
+            "filter": f"and(equals(messages.channel,'{campaign_type}'),greater-or-equal(updated_at,{start_date}),less-than(updated_at,{end_date}))",
+        }
+        url = f"{base_url}?{urlencode(params)}"
+        pages = self._fetch_pages(session, url)
+        for page in pages:
+            page["campaign_type"] = campaign_type
+
+        return pages
+
+    def fetch_tag(self, session: requests.Session):
+        url = f"{BASE_URL}/tags"
+        return self._fetch_pages(session, url, False)
+
+    def fetch_catalog_variant(
+        self,
+        session: requests.Session,
+        last_updated: str,
+    ):
+        url = f"{BASE_URL}/catalog-variants"
+        items = self._fetch_pages(session, url)
+        last_updated_obj = pendulum.parse(last_updated)
+
+        for item in items:
+            updated_at = pendulum.parse(item["updated"])
+            if updated_at > last_updated_obj:
+                yield item
+
+    def fetch_coupons(self, session: requests.Session):
+        url = f"{BASE_URL}/coupons"
+        return self._fetch_pages(session, url, False)
+
+    def fetch_catalog_categories(
+        self,
+        session: requests.Session,
+        last_updated: str,
+    ):
+        url = f"{BASE_URL}/catalog-categories"
+        items = self._fetch_pages(session, url)
+        last_updated_obj = pendulum.parse(last_updated)
+
+        for item in items:
+            updated_at = pendulum.parse(item["updated"])
+            if updated_at > last_updated_obj:
+                yield item
+
+    def fetch_catalog_item(
+        self,
+        session: requests.Session,
+        last_updated: str,
+    ):
+        url = f"{BASE_URL}/catalog-items"
+        items = self._fetch_pages(session, url)
+        last_updated_obj = pendulum.parse(last_updated)
+
+        for item in items:
+            updated_at = pendulum.parse(item["updated"])
+            if updated_at > last_updated_obj:
+                yield item
+
+    def fetch_forms(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+    ):
+        print(f"Fetching forms for {start_date} to {end_date}")
+        url = f"{BASE_URL}/forms/?sort=-updated_at&filter=and(greater-or-equal(updated_at,{start_date}),less-than(updated_at,{end_date}))"
+        return self._fetch_pages(session, url)
+
+    def fetch_lists(
+        self,
+        session: requests.Session,
+        updated_date: str,
+    ):
+        # https://a.klaviyo.com/api/lists/?sort=-updated&filter=greater-than(updated,2024-02-01 00:00:00+00:00)
+        url = f"{BASE_URL}/lists/?sort=-updated&filter=greater-than(updated,{updated_date})"
+        return self._fetch_pages(session, url)
+
+    def fetch_images(self, session: requests.Session, start_date: str, end_date: str):
+        # https://a.klaviyo.com/api/images/?sort=-updated_at&filter=greater-or-equal(updated_at,2024-06-01 00:00:00+00:00),less-than(updated_at,2024-09-01 00:00:00+00:00)
+        url = f"{BASE_URL}/images/?sort=-updated_at&filter=and(greater-or-equal(updated_at,{start_date}),less-than(updated_at,{end_date}))"
+        return self._fetch_pages(session, url)
+
+    def fetch_segments(
+        self,
+        session: requests.Session,
+        updated_date: str,
+    ):
+        # https://a.klaviyo.com/api/segments/?sort=-updated&filter=greater-than(updated,2024-04-01 00:00:00+00:00)
+        url = f"{BASE_URL}/segments/?sort=-updated&filter=greater-than(updated,{updated_date})"
+        print("url", url)
+        return self._fetch_pages(session, url)
+
+    def fetch_flows(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+    ):
+        print(f"Fetching events for {start_date} to {end_date}")
+        # https://a.klaviyo.com/api/flows/?sort=-updated&filter=and(greater-or-equal(updated,2024-06-01 00:00:00+00:00),less-than(updated,2024-09-01 00:00:00+00:00))
+        url = f"{BASE_URL}/flows/?sort=-updated&filter=and(greater-or-equal(updated,{start_date}),less-than(updated,{end_date}))"
+        return self._fetch_pages(session, url)
+
+    def fetch_templates(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+    ):
+        # https://a.klaviyo.com/api/templates/?sort=-updated&filter=and(greater-or-equal(updated,2024-06-01 00:00:00+00:00),less-than(updated,2024-09-01 00:00:00+00:00))
+        url = f"{BASE_URL}/templates/?sort=-updated&filter=and(greater-or-equal(updated,{start_date}),less-than(updated,{end_date}))"
+        return self._fetch_pages(session, url)

ingestr/src/klaviyo/helpers.py

@@ -0,0 +1,19 @@
+from typing import List
+
+import pendulum
+
+
+def split_date_range(
+    start_date: pendulum.DateTime, end_date: pendulum.DateTime
+) -> List[tuple]:
+    interval = "days"
+    if (end_date - start_date).days <= 1:
+        interval = "hours"
+
+    intervals = []
+    current = start_date
+    while current < end_date:
+        next_date = min(current.add(**{interval: 1}), end_date)
+        intervals.append((current.isoformat(), next_date.isoformat()))
+        current = next_date
+    return intervals

ingestr/src/slack/__init__.py

@@ -0,0 +1,272 @@
+"""Fetches Slack Conversations, History and logs."""
+
+from functools import partial
+from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple
+
+import dlt
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+from pendulum import DateTime
+
+from .helpers import SlackAPI, ensure_dt_type
+from .settings import (
+    DEFAULT_DATETIME_FIELDS,
+    DEFAULT_START_DATE,
+    MAX_PAGE_SIZE,
+    MSG_DATETIME_FIELDS,
+)
+
+
+@dlt.source(name="slack", max_table_nesting=0)
+def slack_source(
+    page_size: int = MAX_PAGE_SIZE,
+    access_token: str = dlt.secrets.value,
+    start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+    end_date: Optional[TAnyDateTime] = None,
+    selected_channels: Optional[List[str]] = dlt.config.value,
+    table_per_channel: bool = True,
+    replies: bool = False,
+) -> Iterable[DltResource]:
+    """
+    The source for the Slack pipeline. Available resources are conversations, conversations_history
+    and access_logs.
+
+    Args:
+        page_size: The max number of items to fetch per page. Defaults to 1000.
+        access_token: the oauth access_token used to authenticate.
+        start_date: The start time of the range for which to load. Defaults to January 1st 2000.
+        end_date: The end time of the range for which to load data.
+        selected_channels: The list of channels to load. If None, all channels will be loaded.
+        table_per_channel: Boolean flag, True by default. If True - for each channel separate table with messages is created.
+            Otherwise, all messages are put in one table.
+        replies: Boolean flag indicating if you want a replies table to be present as well. False by default.
+
+    Returns:
+        Iterable[DltResource]: A list of DltResource objects representing the data resources.
+    """
+
+    end_dt: Optional[DateTime] = ensure_dt_type(end_date)
+    start_dt: Optional[DateTime] = ensure_dt_type(start_date)
+    write_disposition: Literal["append", "merge"] = (
+        "append" if end_date is None else "merge"
+    )
+
+    api = SlackAPI(
+        access_token=access_token,
+        page_size=page_size,
+    )
+
+    def get_channels(
+        slack_api: SlackAPI, selected_channels: Optional[List[str]]
+    ) -> Tuple[List[TDataItem], List[TDataItem]]:
+        """
+        Returns channel fetched from slack and list of selected channels.
+
+        Args:
+            slack_api: Slack API instance.
+            selected_channels: List of selected channels names or None.
+
+        Returns:
+            Tuple[List[TDataItem], List[TDataItem]]: fetched channels and selected fetched channels.
+        """
+        channels: List[TDataItem] = []
+        for page_data in slack_api.get_pages(
+            resource="conversations.list",
+            response_path="$.channels[*]",
+            datetime_fields=DEFAULT_DATETIME_FIELDS,
+        ):
+            channels.extend(page_data)
+
+        if selected_channels:
+            fetch_channels = [
+                c
+                for c in channels
+                if c["name"] in selected_channels or c["id"] in selected_channels
+            ]
+        else:
+            fetch_channels = channels
+        return channels, fetch_channels
+
+    channels, fetched_selected_channels = get_channels(api, selected_channels)
+
+    @dlt.resource(name="channels", primary_key="id", write_disposition="replace")
+    def channels_resource() -> Iterable[TDataItem]:
+        """Yield all channels as a DLT resource."""
+        yield from channels
+
+    @dlt.resource(name="users", primary_key="id", write_disposition="replace")
+    def users_resource() -> Iterable[TDataItem]:
+        """
+        Yield all users as a DLT resource.
+
+        Yields:
+            Iterable[TDataItem]: A list of users.
+        """
+
+        for page_data in api.get_pages(
+            resource="users.list",
+            response_path="$.members[*]",
+            params=dict(include_locale=True),
+            datetime_fields=DEFAULT_DATETIME_FIELDS,
+        ):
+            yield page_data
+
+    def get_messages(
+        channel_data: Dict[str, Any], start_date_ts: float, end_date_ts: float
+    ) -> Iterable[TDataItem]:
+        """
+        Generator, which gets channel messages for specific dates.
+        Args:
+            channel_data: dict with channels data.
+            start_date_ts: start timestamp.
+            end_date_ts: end timestamp.
+
+        Yields:
+            List[TDataItem]: messages.
+        """
+        params = {
+            "channel": channel_data["id"],
+            "oldest": start_date_ts,
+            "latest": end_date_ts,
+        }
+
+        for page_data in api.get_pages(
+            resource="conversations.history",
+            response_path="$.messages[*]",
+            params=params,
+            datetime_fields=MSG_DATETIME_FIELDS,
+            context={"channel": channel_data["id"]},
+        ):
+            yield page_data
+
+    def get_thread_replies(messages: List[Dict[str, Any]]) -> Iterable[TDataItem]:
+        """
+        Generator, which gets replies for each message.
+        Args:
+            messages: messages data.
+
+        Yields:
+            Li
+        """
+        for message in messages:
+            if message.get("thread_ts", None):
+                params = {
+                    "channel": message["channel"],
+                    "ts": ensure_dt_type(message["thread_ts"], to_ts=True),
+                }
+
+                for page_data in api.get_pages(
+                    resource="conversations.replies",
+                    response_path="$.messages[*]",
+                    params=params,
+                    context={"channel": message["channel"]},
+                ):
+                    yield page_data[1:]
+
+    @dlt.resource(
+        name="messages",
+        primary_key=("channel", "ts"),
+        columns={"blocks": {"data_type": "complex"}},
+        write_disposition=write_disposition,
+    )
+    def messages_resource(
+        created_at: dlt.sources.incremental[DateTime] = dlt.sources.incremental(
+            "ts",
+            initial_value=start_dt,
+            end_value=end_dt,
+            allow_external_schedulers=True,
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Yield all messages for a set of selected channels as a DLT resource. Keep blocks column without normalization.
+
+        Args:
+            created_at (dlt.sources.incremental[DateTime]): The incremental created_at field.
+
+        Yields:
+            Iterable[TDataItem]: A list of messages.
+        """
+        start_date_ts = ensure_dt_type(created_at.last_value, to_ts=True)
+        end_date_ts = ensure_dt_type(created_at.end_value, to_ts=True)
+        for channel_data in fetched_selected_channels:
+            yield from get_messages(channel_data, start_date_ts, end_date_ts)
+
+    def per_table_messages_resource(
+        channel_data: Dict[str, Any],
+        created_at: dlt.sources.incremental[DateTime] = dlt.sources.incremental(
+            "ts",
+            initial_value=start_dt,
+            end_value=end_dt,
+            allow_external_schedulers=True,
+        ),
+    ) -> Iterable[TDataItem]:
+        """Yield all messages for a given channel as a DLT resource. Keep blocks column without normalization.
+
+        Args:
+            channel_data (Dict[str, Any]): The channel data.
+            created_at (dlt.sources.incremental[DateTime]): The incremental created_at field.
+
+        Yields:
+            Iterable[TDataItem]: A list of messages.
+        """
+        start_date_ts = ensure_dt_type(created_at.last_value, to_ts=True)
+        end_date_ts = ensure_dt_type(created_at.end_value, to_ts=True)
+        yield from get_messages(channel_data, start_date_ts, end_date_ts)
+
+    def table_name_func(channel_name: str, payload: TDataItem) -> str:
+        """Return the table name for a given channel and payload."""
+        table_type = payload.get("subtype", payload.get("type", ""))
+        return f"{channel_name}_{table_type}"
+
+    # It will not work in the pipeline or tests because it is a paid feature,
+    # raise an error when it is not a paying account.
+    @dlt.resource(
+        name="access_logs",
+        selected=False,
+        primary_key="user_id",
+        write_disposition="append",
+    )
+    # it is not an incremental resource it just has an end_date filter
+    def logs_resource() -> Iterable[TDataItem]:
+        """The access logs resource."""
+        for page_data in api.get_pages(
+            resource="team.accessLogs",
+            response_path="$.logins[*]",
+            datetime_fields=["date_first", "date_last"],
+            params={"before": end_dt if end_dt is None else end_dt.int_timestamp},
+        ):
+            yield page_data
+
+    yield from (channels_resource, users_resource, logs_resource)
+
+    if table_per_channel:
+        for channel in fetched_selected_channels:
+            channel_name = channel["name"]
+            table_name = partial(table_name_func, channel_name)
+            messages_channel = dlt.resource(
+                per_table_messages_resource,
+                name=channel_name,
+                table_name=table_name,
+                primary_key=("channel", "ts"),
+                write_disposition=write_disposition,
+                columns={"blocks": {"data_type": "complex"}},
+            )(channel)
+
+            yield messages_channel
+            if replies:
+                yield messages_channel | dlt.transformer(
+                    get_thread_replies,
+                    name=channel_name + "_replies",
+                    table_name=partial(table_name_func, channel_name + "_replies"),
+                    primary_key=("thread_ts", "ts"),
+                    write_disposition=write_disposition,
+                )
+    else:
+        yield messages_resource
+        if replies:
+            yield messages_resource | dlt.transformer(
+                get_thread_replies,
+                name="replies",
+                primary_key=("thread_ts", "ts"),
+                write_disposition=write_disposition,
+            )