omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/klaviyo/__init__.py

@@ -0,0 +1,237 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+import requests
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers.requests import Client
+
+from omniload.src.klaviyo.client import KlaviyoClient
+from omniload.src.klaviyo.helpers import split_date_range
+
+
+def retry_on_limit(response: requests.Response, exception: BaseException) -> bool:
+    return response.status_code == 429
+
+
+def create_client() -> requests.Session:
+    return Client(
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+        request_backoff_factor=2,
+    ).session
+
+
+@dlt.source(max_table_nesting=0)
+def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResource]:
+    start_date_obj = ensure_pendulum_datetime(start_date)
+    client = KlaviyoClient(api_key)
+
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
+    def events(
+        datetime=dlt.sources.incremental(
+            "datetime",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(datetime.start_value), pendulum.now()
+        )
+
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_events(create_client(), s, e)
+
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
+    def profiles(
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated.start_value), pendulum.now()
+        )
+
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_profiles(create_client(), s, e)
+
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
+    def campaigns(
+        updated_at=dlt.sources.incremental(
+            "updated_at",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated_at.start_value), pendulum.now()
+        )
+
+        for campaign_type in ["email", "sms"]:
+            for start, end in intervals:
+                yield lambda s=start, e=end, ct=campaign_type: client.fetch_campaigns(
+                    create_client(), s, e, ct
+                )
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def metrics(
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_metrics(create_client(), updated.start_value)
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def tags() -> Iterable[TAnyDateTime]:
+        yield from client.fetch_tag(create_client())
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def coupons() -> Iterable[TAnyDateTime]:
+        yield from client.fetch_coupons(create_client())
+
+    @dlt.resource(write_disposition="merge", primary_key="id", name="catalog-variants")
+    def catalog_variants(
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_catalog_variant(create_client(), updated.start_value)
+
+    @dlt.resource(
+        write_disposition="merge", primary_key="id", name="catalog-categories"
+    )
+    def catalog_categories(
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_catalog_categories(create_client(), updated.start_value)
+
+    @dlt.resource(write_disposition="merge", primary_key="id", name="catalog-items")
+    def catalog_items(
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_catalog_item(create_client(), updated.start_value)
+
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
+    def forms(
+        updated_at=dlt.sources.incremental(
+            "updated_at",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated_at.start_value), pendulum.now()
+        )
+
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_forms(create_client(), s, e)
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def lists(
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_lists(create_client(), updated.start_value)
+
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
+    def images(
+        updated_at=dlt.sources.incremental(
+            "updated_at",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated_at.start_value), pendulum.now()
+        )
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_images(create_client(), s, e)
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def segments(
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        yield from client.fetch_segments(create_client(), updated.start_value)
+
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
+    def flows(
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated.start_value), pendulum.now()
+        )
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_flows(create_client(), s, e)
+
+    @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
+    def templates(
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        intervals = split_date_range(
+            pendulum.parse(updated.start_value), pendulum.now()
+        )
+        for start, end in intervals:
+            yield lambda s=start, e=end: client.fetch_templates(create_client(), s, e)
+
+    return (
+        events,
+        profiles,
+        campaigns,
+        metrics,
+        tags,
+        coupons,
+        catalog_variants,
+        catalog_categories,
+        catalog_items,
+        forms,
+        lists,
+        images,
+        segments,
+        flows,
+        templates,
+    )

omniload/src/klaviyo/client.py

@@ -0,0 +1,212 @@
+from urllib.parse import urlencode
+
+import pendulum
+import requests
+
+BASE_URL = "https://a.klaviyo.com/api"
+
+
+class KlaviyoClient:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+
+    def __get_headers(self):
+        return {
+            "Authorization": f"Klaviyo-API-Key {self.api_key}",
+            "accept": "application/json",
+            "revision": "2024-07-15",
+        }
+
+    def _flatten_attributes(self, items: list):
+        for event in items:
+            if "attributes" not in event:
+                continue
+
+            for attribute_key in event["attributes"]:
+                event[attribute_key] = event["attributes"][attribute_key]
+
+            del event["attributes"]
+        return items
+
+    def _fetch_pages(
+        self, session: requests.Session, url: str, flat: bool = True
+    ) -> list:
+        all_items = []
+        while True:
+            response = session.get(url=url, headers=self.__get_headers())
+            result = response.json()
+            items = result.get("data", [])
+
+            if flat:
+                items = self._flatten_attributes(items)
+
+            all_items.extend(items)
+            nextURL = result.get("links", {}).get("next")
+            if nextURL is None:
+                break
+
+            url = nextURL
+
+        return all_items
+
+    def fetch_events(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+    ):
+        print(f"Fetching events for {start_date} to {end_date}")
+        url = f"{BASE_URL}/events/?sort=-datetime&filter=and(greater-or-equal(datetime,{start_date}),less-than(datetime,{end_date}))"
+        return self._fetch_pages(session, url)
+
+    def fetch_metrics(
+        self,
+        session: requests.Session,
+        last_updated: str,
+    ):
+        print(f"Fetching metrics since {last_updated}")
+        url = f"{BASE_URL}/metrics"
+        items = self._fetch_pages(session, url)
+
+        last_updated_obj = pendulum.parse(last_updated)
+        for item in items:
+            updated_at = pendulum.parse(item["updated"])
+            if updated_at > last_updated_obj:
+                yield item
+
+    def fetch_profiles(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+    ):
+        pendulum_start_date = pendulum.parse(start_date)
+        pendulum_start_date = pendulum_start_date.subtract(seconds=1)
+        url = f"{BASE_URL}/profiles/?sort=updated&filter=and(greater-than(updated,{pendulum_start_date.isoformat()}),less-than(updated,{end_date}))"
+        return self._fetch_pages(session, url)
+
+    def fetch_campaigns(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+        campaign_type: str,
+    ):
+        print(f"Fetching {campaign_type} campaigns for {start_date} to {end_date}")
+
+        base_url = f"{BASE_URL}/campaigns/"
+        params = {
+            "sort": "updated_at",
+            "filter": f"and(equals(messages.channel,'{campaign_type}'),greater-or-equal(updated_at,{start_date}),less-than(updated_at,{end_date}))",
+        }
+        url = f"{base_url}?{urlencode(params)}"
+        pages = self._fetch_pages(session, url)
+        for page in pages:
+            page["campaign_type"] = campaign_type
+
+        return pages
+
+    def fetch_tag(self, session: requests.Session):
+        url = f"{BASE_URL}/tags"
+        return self._fetch_pages(session, url, False)
+
+    def fetch_catalog_variant(
+        self,
+        session: requests.Session,
+        last_updated: str,
+    ):
+        url = f"{BASE_URL}/catalog-variants"
+        items = self._fetch_pages(session, url)
+        last_updated_obj = pendulum.parse(last_updated)
+
+        for item in items:
+            updated_at = pendulum.parse(item["updated"])
+            if updated_at > last_updated_obj:
+                yield item
+
+    def fetch_coupons(self, session: requests.Session):
+        url = f"{BASE_URL}/coupons"
+        return self._fetch_pages(session, url, False)
+
+    def fetch_catalog_categories(
+        self,
+        session: requests.Session,
+        last_updated: str,
+    ):
+        url = f"{BASE_URL}/catalog-categories"
+        items = self._fetch_pages(session, url)
+        last_updated_obj = pendulum.parse(last_updated)
+
+        for item in items:
+            updated_at = pendulum.parse(item["updated"])
+            if updated_at > last_updated_obj:
+                yield item
+
+    def fetch_catalog_item(
+        self,
+        session: requests.Session,
+        last_updated: str,
+    ):
+        url = f"{BASE_URL}/catalog-items"
+        items = self._fetch_pages(session, url)
+        last_updated_obj = pendulum.parse(last_updated)
+
+        for item in items:
+            updated_at = pendulum.parse(item["updated"])
+            if updated_at > last_updated_obj:
+                yield item
+
+    def fetch_forms(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+    ):
+        print(f"Fetching forms for {start_date} to {end_date}")
+        url = f"{BASE_URL}/forms/?sort=-updated_at&filter=and(greater-or-equal(updated_at,{start_date}),less-than(updated_at,{end_date}))"
+        return self._fetch_pages(session, url)
+
+    def fetch_lists(
+        self,
+        session: requests.Session,
+        updated_date: str,
+    ):
+        # https://a.klaviyo.com/api/lists/?sort=-updated&filter=greater-than(updated,2024-02-01 00:00:00+00:00)
+        url = f"{BASE_URL}/lists/?sort=-updated&filter=greater-than(updated,{updated_date})"
+        return self._fetch_pages(session, url)
+
+    def fetch_images(self, session: requests.Session, start_date: str, end_date: str):
+        # https://a.klaviyo.com/api/images/?sort=-updated_at&filter=greater-or-equal(updated_at,2024-06-01 00:00:00+00:00),less-than(updated_at,2024-09-01 00:00:00+00:00)
+        url = f"{BASE_URL}/images/?sort=-updated_at&filter=and(greater-or-equal(updated_at,{start_date}),less-than(updated_at,{end_date}))"
+        return self._fetch_pages(session, url)
+
+    def fetch_segments(
+        self,
+        session: requests.Session,
+        updated_date: str,
+    ):
+        # https://a.klaviyo.com/api/segments/?sort=-updated&filter=greater-than(updated,2024-04-01 00:00:00+00:00)
+        url = f"{BASE_URL}/segments/?sort=-updated&filter=greater-than(updated,{updated_date})"
+        print("url", url)
+        return self._fetch_pages(session, url)
+
+    def fetch_flows(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+    ):
+        print(f"Fetching events for {start_date} to {end_date}")
+        # https://a.klaviyo.com/api/flows/?sort=-updated&filter=and(greater-or-equal(updated,2024-06-01 00:00:00+00:00),less-than(updated,2024-09-01 00:00:00+00:00))
+        url = f"{BASE_URL}/flows/?sort=-updated&filter=and(greater-or-equal(updated,{start_date}),less-than(updated,{end_date}))"
+        return self._fetch_pages(session, url)
+
+    def fetch_templates(
+        self,
+        session: requests.Session,
+        start_date: str,
+        end_date: str,
+    ):
+        # https://a.klaviyo.com/api/templates/?sort=-updated&filter=and(greater-or-equal(updated,2024-06-01 00:00:00+00:00),less-than(updated,2024-09-01 00:00:00+00:00))
+        url = f"{BASE_URL}/templates/?sort=-updated&filter=and(greater-or-equal(updated,{start_date}),less-than(updated,{end_date}))"
+        return self._fetch_pages(session, url)

omniload/src/klaviyo/helpers.py

@@ -0,0 +1,19 @@
+from typing import List
+
+import pendulum
+
+
+def split_date_range(
+    start_date: pendulum.DateTime, end_date: pendulum.DateTime
+) -> List[tuple]:
+    interval = "days"
+    if (end_date - start_date).days <= 1:
+        interval = "hours"
+
+    intervals = []
+    current = start_date
+    while current < end_date:
+        next_date = min(current.add(**{interval: 1}), end_date)
+        intervals.append((current.isoformat(), next_date.isoformat()))
+        current = next_date
+    return intervals