airbyte-source-iterable 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

source_iterable/source.py

@@ -2,20 +2,14 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from typing import Any, List, Mapping, Tuple
+from typing import Any, List, Mapping
 
-import requests.exceptions
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
-from source_iterable.utils import read_full_refresh
 
 from .streams import (
-    AccessCheck,
-    Campaigns,
     CampaignsMetrics,
-    Channels,
     CustomEvent,
     EmailBounce,
     EmailClick,
@@ -25,7 +19,6 @@ from .streams import (
     EmailSendSkip,
     EmailSubscribe,
     EmailUnsubscribe,
-    Events,
     HostedUnsubscribeClick,
     InAppClick,
     InAppClose,
@@ -36,10 +29,6 @@ from .streams import (
     InAppSendSkip,
     InboxMessageImpression,
     InboxSession,
-    Lists,
-    ListUsers,
-    MessageTypes,
-    Metadata,
     Purchase,
     PushBounce,
     PushOpen,
@@ -53,101 +42,70 @@ from .streams import (
     SmsSendSkip,
     SmsUsageInfo,
     Templates,
-    Users,
     WebPushClick,
     WebPushSend,
     WebPushSendSkip,
 )
 
+"""
+This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
+source connector.
 
-class SourceIterable(AbstractSource):
-    """
-    Note: there are some redundant endpoints
-    (e.g. [`export/userEvents`](https://api.iterable.com/api/docs#export_exportUserEvents)
-    and [`events/{email}`](https://api.iterable.com/api/docs#events_User_events)).
-    In this case it's better to use the one which takes params as a query param rather than as part of the url param.
-    """
+WARNING: Do not modify this file.
+"""
 
-    def check_connection(self, logger, config) -> Tuple[bool, any]:
-        try:
-            authenticator = TokenAuthenticator(token=config["api_key"], auth_header="Api-Key", auth_method="")
-            list_gen = Lists(authenticator=authenticator).read_records(sync_mode=SyncMode.full_refresh)
-            next(list_gen)
-            return True, None
-        except Exception as e:
-            return False, f"Unable to connect to Iterable API with the provided credentials - {e}"
+# Declarative Source
+class SourceIterable(YamlDeclarativeSource):
+    def __init__(self):
+        super().__init__(**{"path_to_yaml": "manifest.yaml"})
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        def all_streams_accessible():
-            access_check_stream = AccessCheck(authenticator=authenticator)
-            try:
-                next(read_full_refresh(access_check_stream), None)
-            except requests.exceptions.RequestException as e:
-                if e.response.status_code == requests.codes.UNAUTHORIZED:
-                    return False
-                raise
-            return True
+        streams = super().streams(config=config)
 
         authenticator = TokenAuthenticator(token=config["api_key"], auth_header="Api-Key", auth_method="")
         # end date is provided for integration tests only
         start_date, end_date = config["start_date"], config.get("end_date")
         date_range = {"start_date": start_date, "end_date": end_date}
-        streams = [
-            Campaigns(authenticator=authenticator),
-            CampaignsMetrics(authenticator=authenticator, **date_range),
-            Channels(authenticator=authenticator),
-            Lists(authenticator=authenticator),
-            MessageTypes(authenticator=authenticator),
-            Metadata(authenticator=authenticator),
-            Templates(authenticator=authenticator, **date_range),
-        ]
-        # Iterable supports two types of Server-side api keys:
-        # - read only
-        # - server side
-        # The first one has a limited set of supported APIs, so others are filtered out here.
-        # A simple check is done - a read operation on a stream that can be accessed only via a Server side API key.
-        # If read is successful - other streams should be supported as well.
-        # More on this - https://support.iterable.com/hc/en-us/articles/360043464871-API-Keys-
-        if all_streams_accessible():
-            streams.extend(
-                [
-                    Users(authenticator=authenticator, **date_range),
-                    ListUsers(authenticator=authenticator),
-                    EmailBounce(authenticator=authenticator, **date_range),
-                    EmailClick(authenticator=authenticator, **date_range),
-                    EmailComplaint(authenticator=authenticator, **date_range),
-                    EmailOpen(authenticator=authenticator, **date_range),
-                    EmailSend(authenticator=authenticator, **date_range),
-                    EmailSendSkip(authenticator=authenticator, **date_range),
-                    EmailSubscribe(authenticator=authenticator, **date_range),
-                    EmailUnsubscribe(authenticator=authenticator, **date_range),
-                    PushSend(authenticator=authenticator, **date_range),
-                    PushSendSkip(authenticator=authenticator, **date_range),
-                    PushOpen(authenticator=authenticator, **date_range),
-                    PushUninstall(authenticator=authenticator, **date_range),
-                    PushBounce(authenticator=authenticator, **date_range),
-                    WebPushSend(authenticator=authenticator, **date_range),
-                    WebPushClick(authenticator=authenticator, **date_range),
-                    WebPushSendSkip(authenticator=authenticator, **date_range),
-                    InAppSend(authenticator=authenticator, **date_range),
-                    InAppOpen(authenticator=authenticator, **date_range),
-                    InAppClick(authenticator=authenticator, **date_range),
-                    InAppClose(authenticator=authenticator, **date_range),
-                    InAppDelete(authenticator=authenticator, **date_range),
-                    InAppDelivery(authenticator=authenticator, **date_range),
-                    InAppSendSkip(authenticator=authenticator, **date_range),
-                    InboxSession(authenticator=authenticator, **date_range),
-                    InboxMessageImpression(authenticator=authenticator, **date_range),
-                    SmsSend(authenticator=authenticator, **date_range),
-                    SmsBounce(authenticator=authenticator, **date_range),
-                    SmsClick(authenticator=authenticator, **date_range),
-                    SmsReceived(authenticator=authenticator, **date_range),
-                    SmsSendSkip(authenticator=authenticator, **date_range),
-                    SmsUsageInfo(authenticator=authenticator, **date_range),
-                    Purchase(authenticator=authenticator, **date_range),
-                    CustomEvent(authenticator=authenticator, **date_range),
-                    HostedUnsubscribeClick(authenticator=authenticator, **date_range),
-                    Events(authenticator=authenticator),
-                ]
-            )
+
+        # TODO: migrate streams below to low code as slicer logic will be migrated to generator based
+        streams.extend(
+            [
+                CampaignsMetrics(authenticator=authenticator, **date_range),
+                Templates(authenticator=authenticator, **date_range),
+                EmailBounce(authenticator=authenticator, **date_range),
+                EmailClick(authenticator=authenticator, **date_range),
+                EmailComplaint(authenticator=authenticator, **date_range),
+                EmailOpen(authenticator=authenticator, **date_range),
+                EmailSend(authenticator=authenticator, **date_range),
+                EmailSendSkip(authenticator=authenticator, **date_range),
+                EmailSubscribe(authenticator=authenticator, **date_range),
+                EmailUnsubscribe(authenticator=authenticator, **date_range),
+                PushSend(authenticator=authenticator, **date_range),
+                PushSendSkip(authenticator=authenticator, **date_range),
+                PushOpen(authenticator=authenticator, **date_range),
+                PushUninstall(authenticator=authenticator, **date_range),
+                PushBounce(authenticator=authenticator, **date_range),
+                WebPushSend(authenticator=authenticator, **date_range),
+                WebPushClick(authenticator=authenticator, **date_range),
+                WebPushSendSkip(authenticator=authenticator, **date_range),
+                InAppSend(authenticator=authenticator, **date_range),
+                InAppOpen(authenticator=authenticator, **date_range),
+                InAppClick(authenticator=authenticator, **date_range),
+                InAppClose(authenticator=authenticator, **date_range),
+                InAppDelete(authenticator=authenticator, **date_range),
+                InAppDelivery(authenticator=authenticator, **date_range),
+                InAppSendSkip(authenticator=authenticator, **date_range),
+                InboxSession(authenticator=authenticator, **date_range),
+                InboxMessageImpression(authenticator=authenticator, **date_range),
+                SmsSend(authenticator=authenticator, **date_range),
+                SmsBounce(authenticator=authenticator, **date_range),
+                SmsClick(authenticator=authenticator, **date_range),
+                SmsReceived(authenticator=authenticator, **date_range),
+                SmsSendSkip(authenticator=authenticator, **date_range),
+                SmsUsageInfo(authenticator=authenticator, **date_range),
+                Purchase(authenticator=authenticator, **date_range),
+                CustomEvent(authenticator=authenticator, **date_range),
+                HostedUnsubscribeClick(authenticator=authenticator, **date_range),
+            ]
+        )
         return streams

source_iterable/spec.json

@@ -1,5 +1,4 @@
 {
-  "documentationUrl": "https://docs.airbyte.com/integrations/sources/iterable",
   "connectionSpecification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Iterable Spec",
@@ -10,14 +9,14 @@
       "api_key": {
         "type": "string",
         "title": "API Key",
-        "description": "Iterable API Key. See the <a href=\"https://docs.airbyte.com/integrations/sources/iterable\">docs</a> for more information on how to obtain this key.",
+        "description": "Iterable API Key. See the <a href=\\\"https://docs.airbyte.com/integrations/sources/iterable\\\">docs</a>  for more information on how to obtain this key.",
         "airbyte_secret": true,
         "order": 0
       },
       "start_date": {
         "type": "string",
         "title": "Start Date",
-        "description": "The date from which you'd like to replicate data for Iterable, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.",
+        "description": "The date from which you'd like to replicate data for Iterable, in the format YYYY-MM-DDT00:00:00Z.  All data generated after this date will be replicated.",
         "examples": ["2021-04-01T00:00:00Z"],
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
         "order": 1,

source_iterable/streams.py

@@ -4,7 +4,6 @@
 
 import csv
 import json
-import urllib.parse as urlparse
 from abc import ABC, abstractmethod
 from io import StringIO
 from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union
@@ -66,11 +65,6 @@ class IterableStream(HttpStream, ABC):
         """
         return None
 
-    def check_unauthorized_key(self, response: requests.Response) -> bool:
-        if response.status_code == codes.UNAUTHORIZED:
-            self.logger.warning(f"Provided API Key has not sufficient permissions to read from stream: {self.data_field}")
-            return True
-
     def check_generic_error(self, response: requests.Response) -> bool:
         """
         https://github.com/airbytehq/oncall/issues/1592#issuecomment-1499109251
@@ -129,9 +123,6 @@ class IterableStream(HttpStream, ABC):
             yield from super().read_records(sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state)
         except (HTTPError, UserDefinedBackoffException, DefaultBackoffException) as e:
             response = e.response
-            if self.check_unauthorized_key(response):
-                self.ignore_further_slices = True
-                return
             if self.check_generic_error(response):
                 return
             raise e
@@ -342,42 +333,6 @@ class IterableExportEventsStreamAdjustableRange(IterableExportStreamAdjustableRa
         return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("events")
 
 
-class Lists(IterableStream):
-    data_field = "lists"
-
-    def path(self, **kwargs) -> str:
-        return "lists"
-
-
-class ListUsers(IterableStream):
-    primary_key = "listId"
-    data_field = "getUsers"
-    name = "list_users"
-    # enable caching, because this stream used by other ones
-    use_cache = True
-
-    def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str:
-        return f"lists/{self.data_field}?listId={stream_slice['list_id']}"
-
-    def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
-        lists = Lists(authenticator=self._cred)
-        for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh)):
-            yield {"list_id": list_record["id"]}
-
-    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        list_id = self._get_list_id(response.url)
-        for user in response.iter_lines():
-            yield {"email": user.decode(), "listId": list_id}
-
-    @staticmethod
-    def _get_list_id(url: str) -> int:
-        parsed_url = urlparse.urlparse(url)
-        for q in parsed_url.query.split("&"):
-            key, value = q.split("=")
-            if key == "listId":
-                return int(value)
-
-
 class Campaigns(IterableStream):
     data_field = "campaigns"
 
@@ -465,71 +420,6 @@ class CampaignsMetrics(IterableStream):
         return result
 
 
-class Channels(IterableStream):
-    data_field = "channels"
-
-    def path(self, **kwargs) -> str:
-        return "channels"
-
-
-class MessageTypes(IterableStream):
-    data_field = "messageTypes"
-    name = "message_types"
-
-    def path(self, **kwargs) -> str:
-        return "messageTypes"
-
-
-class Metadata(IterableStream):
-    primary_key = None
-    data_field = "results"
-
-    def path(self, **kwargs) -> str:
-        return "metadata"
-
-
-class Events(IterableStream):
-    """
-    https://api.iterable.com/api/docs#export_exportUserEvents
-    """
-
-    primary_key = None
-    data_field = "events"
-    common_fields = ("itblInternal", "_type", "createdAt", "email")
-
-    def path(self, **kwargs) -> str:
-        return "export/userEvents"
-
-    def request_params(self, stream_slice: Optional[Mapping[str, Any]], **kwargs) -> MutableMapping[str, Any]:
-        params = super().request_params(**kwargs)
-        params.update({"email": stream_slice["email"], "includeCustomEvents": "true"})
-
-        return params
-
-    def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
-        lists = ListUsers(authenticator=self._cred)
-        stream_slices = lists.stream_slices()
-
-        for stream_slice in stream_slices:
-            for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh), stream_slice=stream_slice):
-                yield {"email": list_record["email"]}
-
-    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        """
-        Parse jsonl response body.
-        Put common event fields at the top level.
-        Put the rest of the fields in the `data` subobject.
-        """
-        jsonl_records = StringIO(response.text)
-        for record in jsonl_records:
-            record_dict = json.loads(record)
-            record_dict_common_fields = {}
-            for field in self.common_fields:
-                record_dict_common_fields[field] = record_dict.pop(field, None)
-
-            yield {**record_dict_common_fields, "data": record_dict}
-
-
 class EmailBounce(IterableExportStreamAdjustableRange):
     data_field = "emailBounce"
 
@@ -687,16 +577,3 @@ class Templates(IterableExportStreamRanged):
         for record in records:
             record[self.cursor_field] = self._field_to_datetime(record[self.cursor_field])
             yield record
-
-
-class Users(IterableExportStreamRanged):
-    data_field = "user"
-    cursor_field = "profileUpdatedAt"
-
-
-class AccessCheck(ListUsers):
-    # since 401 error is failed silently in all the streams,
-    # we need another class to distinguish an empty stream from 401 response
-    def check_unauthorized_key(self, response: requests.Response) -> bool:
-        # this allows not retrying 401 and raising the error upstream
-        return response.status_code != codes.UNAUTHORIZED

source_iterable/utils.py

@@ -24,10 +24,3 @@ def dateutil_parse(text):
         dt.microsecond,
         tz=dt.tzinfo or pendulum.tz.UTC,
     )
-
-
-def read_full_refresh(stream_instance: Stream):
-    slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh)
-    for _slice in slices:
-        for record in stream_instance.read_records(stream_slice=_slice, sync_mode=SyncMode.full_refresh):
-            yield record

airbyte_source_iterable-0.4.0.dist-info/RECORD

@@ -1,29 +0,0 @@
-source_iterable/__init__.py,sha256=8WKQT800ggxG1vGPoA_ZHUkozwEM7qMhGMhzEPCdA4o,65
-source_iterable/run.py,sha256=-Bvz772Z7iJWExDoJS7chxv725hG4e9e5Engrtp66iE,236
-source_iterable/schemas/campaigns.json,sha256=3sDq9ekGVuIoAb0kXhul-5sbaz1Vqm36MnLr0cNfPpY,1191
-source_iterable/schemas/campaigns_metrics.json,sha256=huOj1N5iXi-jvM0ztAZjf2mUIfd2KeWKSwQ7pNmsx6g,109
-source_iterable/schemas/channels.json,sha256=xKQgmDcVeRcN0TgqUY1oEFTjXeHdVeUjCix7OnnjkLg,285
-source_iterable/schemas/email_bounce.json,sha256=GgVv_AesrbRypkePrM1WH5upCYyTffhxxFksG-tSvCo,815
-source_iterable/schemas/email_click.json,sha256=IwQ2GfqhQuefPNDIzbf8Dx7WjxtzcabnlRgTEAA57e4,1267
-source_iterable/schemas/email_complaint.json,sha256=GgVv_AesrbRypkePrM1WH5upCYyTffhxxFksG-tSvCo,815
-source_iterable/schemas/email_open.json,sha256=Vhd2CH2nYXNiLF9pWMiz3GyG82kml1VGkQY1YgW2AI4,1094
-source_iterable/schemas/email_send.json,sha256=DPf-L2VWjyI_iLWNgTp9TZQNi6ee7Pjha39Hmodc5WQ,3027
-source_iterable/schemas/email_send_skip.json,sha256=6aVemp9UpFjyAJbavlSq0r15KrU4sJUj_Wks2VedkEE,3021
-source_iterable/schemas/email_subscribe.json,sha256=LuFhVl3Vz0IbZM0wpNp29KIAyTJQYE_GDNRMy5o2Mws,930
-source_iterable/schemas/email_unsubscribe.json,sha256=E4v8BrTrhKvaMb9wHU55mk8kWwL4vG4I7cytjFUneiE,1153
-source_iterable/schemas/events.json,sha256=pOFyNytQlWMiTkeGzMlDesPc1Miop8AlGBltd95cCL8,679
-source_iterable/schemas/list_users.json,sha256=Z3ff2_IEaJeDNuudl7SJVnj3eBCuM552ts4JT0qNENE,167
-source_iterable/schemas/lists.json,sha256=iNY9jN7k4LwxnQ7OhAl99gQf-CIzcFRKC4cNoJVB6D4,280
-source_iterable/schemas/message_types.json,sha256=GLFtrZk2vkBhnk-OMnQ--cl2VYQNJgPU4a1oBhlZcz4,220
-source_iterable/schemas/metadata.json,sha256=WP_wq49tAsvetXEtQ3KeGxVpqTYCbLjZacNkTohi3jw,109
-source_iterable/schemas/templates.json,sha256=dC7lKUnG3mQe9-Wj-jd7OyYruQEWvYT8DHfW9oDLXT4,569
-source_iterable/schemas/users.json,sha256=WBl7sJBKU3r_BqGF5Nzv0C-rbHj0DoI-PRHopxA7s2k,7490
-source_iterable/slice_generators.py,sha256=neRuWD52Xnr84KvWQ0kjaErR44sOKjxIyLCNeofohzE,6482
-source_iterable/source.py,sha256=RC-3qvp_M6n4vKXsxyyOk8c6tqwh6t7FKX64XeMHcAk,6707
-source_iterable/spec.json,sha256=QQ2V2eUQW88gtgpNjpxOe_zFbBDFIU8zI7Gtv25LC2I,1083
-source_iterable/streams.py,sha256=6x2T0r89IIGsB7iO7suSBBAI96uyFlM4PuTky5MxTek,23693
-source_iterable/utils.py,sha256=ppDgm3BpTDsXY6AsLg3POL3mKj2gkyEmontg86l3ifQ,932
-airbyte_source_iterable-0.4.0.dist-info/METADATA,sha256=d_jPB8qa4zWDkC2qlXhrFPgOvn4wQ-JqwsUvdMxHunw,5350
-airbyte_source_iterable-0.4.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-airbyte_source_iterable-0.4.0.dist-info/entry_points.txt,sha256=pjzrNtsOX3jW37IK0U5pSmLiraiuLTPr5aB5-hBXpAo,59
-airbyte_source_iterable-0.4.0.dist-info/RECORD,,