airbyte-source-iterable 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

{airbyte_source_iterable-0.3.0.dist-info → airbyte_source_iterable-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-iterable
-Version: 0.3.0
+Version: 0.5.0
 Summary: Source implementation for Iterable.
 Home-page: https://airbyte.com
 License: MIT
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (==0.63.2)
+Requires-Dist: airbyte-cdk (>=0,<1)
 Requires-Dist: pendulum (==2.1.2)
 Requires-Dist: python-dateutil (==2.8.2)
 Requires-Dist: requests (==2.31.0)

{airbyte_source_iterable-0.3.0.dist-info → airbyte_source_iterable-0.5.0.dist-info}/RECORD

@@ -1,4 +1,6 @@
 source_iterable/__init__.py,sha256=8WKQT800ggxG1vGPoA_ZHUkozwEM7qMhGMhzEPCdA4o,65
+source_iterable/components.py,sha256=dHp23THc43TmogP_tCxcv12mAgZSRs5fKir2ckr4hHI,1374
+source_iterable/manifest.yaml,sha256=uO7z-bIU6CtGms6IsLwjyZ-cb3tVY-vLUMLOZGkZDts,13996
 source_iterable/run.py,sha256=-Bvz772Z7iJWExDoJS7chxv725hG4e9e5Engrtp66iE,236
 source_iterable/schemas/campaigns.json,sha256=3sDq9ekGVuIoAb0kXhul-5sbaz1Vqm36MnLr0cNfPpY,1191
 source_iterable/schemas/campaigns_metrics.json,sha256=huOj1N5iXi-jvM0ztAZjf2mUIfd2KeWKSwQ7pNmsx6g,109
@@ -19,11 +21,11 @@ source_iterable/schemas/metadata.json,sha256=WP_wq49tAsvetXEtQ3KeGxVpqTYCbLjZacN
 source_iterable/schemas/templates.json,sha256=dC7lKUnG3mQe9-Wj-jd7OyYruQEWvYT8DHfW9oDLXT4,569
 source_iterable/schemas/users.json,sha256=WBl7sJBKU3r_BqGF5Nzv0C-rbHj0DoI-PRHopxA7s2k,7490
 source_iterable/slice_generators.py,sha256=neRuWD52Xnr84KvWQ0kjaErR44sOKjxIyLCNeofohzE,6482
-source_iterable/source.py,sha256=RC-3qvp_M6n4vKXsxyyOk8c6tqwh6t7FKX64XeMHcAk,6707
-source_iterable/spec.json,sha256=QQ2V2eUQW88gtgpNjpxOe_zFbBDFIU8zI7Gtv25LC2I,1083
-source_iterable/streams.py,sha256=6x2T0r89IIGsB7iO7suSBBAI96uyFlM4PuTky5MxTek,23693
-source_iterable/utils.py,sha256=ppDgm3BpTDsXY6AsLg3POL3mKj2gkyEmontg86l3ifQ,932
-airbyte_source_iterable-0.3.0.dist-info/METADATA,sha256=4ecXD0HVzjJ-8rRY7r7AzJYsR4fcyUZECKE1WneAJlI,5352
-airbyte_source_iterable-0.3.0.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-airbyte_source_iterable-0.3.0.dist-info/entry_points.txt,sha256=pjzrNtsOX3jW37IK0U5pSmLiraiuLTPr5aB5-hBXpAo,59
-airbyte_source_iterable-0.3.0.dist-info/RECORD,,
+source_iterable/source.py,sha256=AhZQ4L54VTo6jf2QltvFCqDNgqaPSEKCfR0GInCLEEc,4537
+source_iterable/spec.json,sha256=y5_YFTkPPtxjvzGtaZZql90Tc37fyZZpQKXFGPL_cYE,1009
+source_iterable/streams.py,sha256=ZEC8IqBrTeD2oYOvLL51E62iOsIOK3jiTN7pLI-CwRs,19468
+source_iterable/utils.py,sha256=2oM8AjBZXs9nTG_PhSWmxBWEmp-w-tWFaxQQvAfUuMM,649
+airbyte_source_iterable-0.5.0.dist-info/METADATA,sha256=6K8Owfuh9l-DQAw969Ex5-OalA5t6bkod1ttmDe2qqE,5350
+airbyte_source_iterable-0.5.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+airbyte_source_iterable-0.5.0.dist-info/entry_points.txt,sha256=pjzrNtsOX3jW37IK0U5pSmLiraiuLTPr5aB5-hBXpAo,59
+airbyte_source_iterable-0.5.0.dist-info/RECORD,,

{airbyte_source_iterable-0.3.0.dist-info → airbyte_source_iterable-0.5.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.8.1
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any

source_iterable/components.py

@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import json
+from dataclasses import dataclass
+from io import StringIO
+
+import requests
+from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
+from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState
+
+
+@dataclass
+class XJsonRecordExtractor(DpathExtractor):
+    def extract_records(self, response: requests.Response) -> list[Record]:
+        return [json.loads(record) for record in response.iter_lines()]
+
+
+@dataclass
+class ListUsersRecordExtractor(DpathExtractor):
+    def extract_records(self, response: requests.Response) -> list[Record]:
+        return [{"email": record.decode()} for record in response.iter_lines()]
+
+
+@dataclass
+class EventsRecordExtractor(DpathExtractor):
+    common_fields = ("itblInternal", "_type", "createdAt", "email")
+
+    def extract_records(self, response: requests.Response) -> list[Record]:
+        jsonl_records = StringIO(response.text)
+        records = []
+        for record in jsonl_records:
+            record_dict = json.loads(record)
+            record_dict_common_fields = {}
+            for field in self.common_fields:
+                record_dict_common_fields[field] = record_dict.pop(field, None)
+
+            records.append({**record_dict_common_fields, "data": record_dict})
+
+        return records

source_iterable/manifest.yaml

@@ -0,0 +1,440 @@
+spec:
+  type: Spec
+  connection_specification:
+    $schema: http://json-schema.org/draft-07/schema#
+    title: Iterable Spec
+    type: object
+    required:
+      - start_date
+      - api_key
+    additionalProperties: true
+    properties:
+      api_key:
+        type: "string"
+        title: "API Key"
+        description: >-
+          Iterable API Key. See the <a href=\"https://docs.airbyte.com/integrations/sources/iterable\">docs</a> 
+          for more information on how to obtain this key.
+        airbyte_secret: true
+        order: 0
+      start_date:
+        type: "string"
+        title: "Start Date"
+        description: >-
+          The date from which you'd like to replicate data for Iterable, in the format YYYY-MM-DDT00:00:00Z. 
+          All data generated after this date will be replicated.
+        examples: ["2021-04-01T00:00:00Z"]
+        pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$
+        order: 1
+        format: "date-time"
+type: DeclarativeSource
+check:
+  type: CheckStream
+  stream_names:
+    - lists
+streams:
+  - name: lists
+    type: DeclarativeStream
+    retriever:
+      type: SimpleRetriever
+      paginator:
+        type: NoPagination
+      requester:
+        path: lists
+        type: HttpRequester
+        url_base: https://api.iterable.com/api/
+        http_method: GET
+        authenticator:
+          type: ApiKeyAuthenticator
+          api_token: "{{ config['api_key'] }}"
+          inject_into:
+            type: RequestOption
+            field_name: Api-Key
+            inject_into: header
+        request_headers: {}
+        request_body_json: {}
+        request_parameters: {}
+      record_selector:
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path:
+            - lists
+      partition_router: []
+    primary_key:
+      - id
+  - name: list_users
+    type: DeclarativeStream
+    retriever:
+      type: SimpleRetriever
+      paginator:
+        type: NoPagination
+      requester:
+        path: lists/getUsers
+        type: HttpRequester
+        url_base: https://api.iterable.com/api/
+        http_method: GET
+        authenticator:
+          type: ApiKeyAuthenticator
+          api_token: "{{ config['api_key'] }}"
+          inject_into:
+            type: RequestOption
+            field_name: Api-Key
+            inject_into: header
+        request_headers: {}
+        request_body_json: {}
+        request_parameters: {}
+      record_selector:
+        type: RecordSelector
+        extractor:
+          class_name: source_iterable.components.ListUsersRecordExtractor
+          field_path:
+            - getUsers
+      partition_router:
+        - type: SubstreamPartitionRouter
+          parent_stream_configs:
+            - type: ParentStreamConfig
+              parent_key: id
+              request_option:
+                inject_into: request_parameter
+                type: RequestOption
+                field_name: listId
+              partition_field: list_id
+              stream:
+                type: DeclarativeStream
+                name: lists
+                primary_key:
+                  - id
+                retriever:
+                  type: SimpleRetriever
+                  requester:
+                    type: HttpRequester
+                    url_base: https://api.iterable.com/api/
+                    path: lists
+                    http_method: GET
+                    request_parameters: {}
+                    request_headers: {}
+                    authenticator:
+                      type: ApiKeyAuthenticator
+                      api_token: "{{ config['api_key'] }}"
+                      inject_into:
+                        type: RequestOption
+                        field_name: Api-Key
+                        inject_into: header
+                    request_body_json: {}
+                  record_selector:
+                    type: RecordSelector
+                    extractor:
+                      type: DpathExtractor
+                      field_path:
+                        - lists
+                  paginator:
+                    type: NoPagination
+                  partition_router: []
+    primary_key:
+      - listId
+    transformations:
+      - type: AddFields
+        fields:
+          - path:
+              - listId
+            value: "{{ stream_slice.list_id }}"
+  - name: campaigns
+    type: DeclarativeStream
+    retriever:
+      type: SimpleRetriever
+      paginator:
+        type: NoPagination
+      requester:
+        path: campaigns
+        type: HttpRequester
+        url_base: https://api.iterable.com/api/
+        http_method: GET
+        authenticator:
+          type: ApiKeyAuthenticator
+          api_token: "{{ config['api_key'] }}"
+          inject_into:
+            type: RequestOption
+            field_name: Api-Key
+            inject_into: header
+        request_headers: {}
+        request_body_json: {}
+        request_parameters: {}
+      record_selector:
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path:
+            - campaigns
+      partition_router: []
+    primary_key:
+      - id
+  - name: channels
+    type: DeclarativeStream
+    retriever:
+      type: SimpleRetriever
+      paginator:
+        type: NoPagination
+      requester:
+        path: channels
+        type: HttpRequester
+        url_base: https://api.iterable.com/api/
+        http_method: GET
+        authenticator:
+          type: ApiKeyAuthenticator
+          api_token: "{{ config['api_key'] }}"
+          inject_into:
+            type: RequestOption
+            field_name: Api-Key
+            inject_into: header
+        request_headers: {}
+        request_body_json: {}
+        request_parameters: {}
+      record_selector:
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path:
+            - channels
+      partition_router: []
+    primary_key:
+      - id
+  - name: message_types
+    type: DeclarativeStream
+    retriever:
+      type: SimpleRetriever
+      paginator:
+        type: NoPagination
+      requester:
+        path: messageTypes
+        type: HttpRequester
+        url_base: https://api.iterable.com/api/
+        http_method: GET
+        authenticator:
+          type: ApiKeyAuthenticator
+          api_token: "{{ config['api_key'] }}"
+          inject_into:
+            type: RequestOption
+            field_name: Api-Key
+            inject_into: header
+        request_headers: {}
+        request_body_json: {}
+        request_parameters: {}
+      record_selector:
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path:
+            - messageTypes
+      partition_router: []
+    primary_key:
+      - id
+  - name: metadata
+    type: DeclarativeStream
+    retriever:
+      type: SimpleRetriever
+      paginator:
+        type: NoPagination
+      requester:
+        path: metadata
+        type: HttpRequester
+        url_base: https://api.iterable.com/api/
+        http_method: GET
+        authenticator:
+          type: ApiKeyAuthenticator
+          api_token: "{{ config['api_key'] }}"
+          inject_into:
+            type: RequestOption
+            field_name: Api-Key
+            inject_into: header
+        request_headers: {}
+        request_body_json: {}
+        request_parameters: {}
+      record_selector:
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path:
+            - results
+      partition_router: []
+    primary_key: []
+  - name: users
+    type: DeclarativeStream
+    retriever:
+      type: SimpleRetriever
+      paginator:
+        type: NoPagination
+      requester:
+        path: export/data.json
+        type: HttpRequester
+        url_base: https://api.iterable.com/api/
+        http_method: GET
+        authenticator:
+          type: ApiKeyAuthenticator
+          api_token: "{{ config['api_key'] }}"
+          inject_into:
+            type: RequestOption
+            field_name: Api-Key
+            inject_into: header
+        request_headers: {}
+        request_body_json: {}
+        request_parameters:
+          stream: "True"
+          dataTypeName: user
+      record_selector:
+        type: RecordSelector
+        extractor:
+          class_name: source_iterable.components.XJsonRecordExtractor
+          field_path:
+            - users
+      partition_router: []
+    primary_key: []
+    incremental_sync:
+      step: P90D
+      type: DatetimeBasedCursor
+      cursor_field: profileUpdatedAt
+      end_datetime:
+        type: MinMaxDatetime
+        datetime: "{{ config['end_date'] if config['end_date'] else now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}"
+        datetime_format: "%Y-%m-%dT%H:%M:%SZ"
+      start_datetime:
+        type: MinMaxDatetime
+        datetime: "{{ config['start_date'] }}"
+        datetime_format: "%Y-%m-%dT%H:%M:%SZ"
+      datetime_format: "%Y-%m-%d %H:%M:%S"
+      end_time_option:
+        type: RequestOption
+        field_name: endDateTime
+        inject_into: request_parameter
+      start_time_option:
+        type: RequestOption
+        field_name: startDateTime
+        inject_into: request_parameter
+      cursor_granularity: PT1S
+      cursor_datetime_formats:
+        - "%Y-%m-%d %H:%M:%S %z"
+        - "%Y-%m-%dT%H:%M:%S%z"
+  - name: events
+    primary_key: []
+    retriever:
+      type: SimpleRetriever
+      requester:
+        type: HttpRequester
+        url_base: https://api.iterable.com/api/
+        path: export/userEvents
+        http_method: GET
+        request_parameters:
+          includeCustomEvents: "true"
+        request_headers: {}
+        authenticator:
+          type: ApiKeyAuthenticator
+          api_token: "{{ config['api_key'] }}"
+          inject_into:
+            type: RequestOption
+            field_name: Api-Key
+            inject_into: header
+        request_body_json: {}
+      record_selector:
+        type: RecordSelector
+        extractor:
+          class_name: source_iterable.components.EventsRecordExtractor
+          field_path:
+            - events
+      paginator:
+        type: NoPagination
+      partition_router:
+        - type: SubstreamPartitionRouter
+          parent_stream_configs:
+            - type: ParentStreamConfig
+              parent_key: email
+              request_option:
+                inject_into: request_parameter
+                type: RequestOption
+                field_name: email
+              partition_field: email
+              stream:
+                name: list_users
+                type: DeclarativeStream
+                retriever:
+                  type: SimpleRetriever
+                  paginator:
+                    type: NoPagination
+                  requester:
+                    path: lists/getUsers
+                    type: HttpRequester
+                    url_base: https://api.iterable.com/api/
+                    http_method: GET
+                    authenticator:
+                      type: ApiKeyAuthenticator
+                      api_token: "{{ config['api_key'] }}"
+                      inject_into:
+                        type: RequestOption
+                        field_name: Api-Key
+                        inject_into: header
+                    request_headers: {}
+                    request_body_json: {}
+                    request_parameters: {}
+                  record_selector:
+                    type: RecordSelector
+                    extractor:
+                      class_name: source_iterable.components.ListUsersRecordExtractor
+                      field_path:
+                        - getUsers
+                  partition_router:
+                    - type: SubstreamPartitionRouter
+                      parent_stream_configs:
+                        - type: ParentStreamConfig
+                          parent_key: id
+                          request_option:
+                            inject_into: request_parameter
+                            type: RequestOption
+                            field_name: listId
+                          partition_field: list_id
+                          stream:
+                            type: DeclarativeStream
+                            name: lists
+                            primary_key:
+                              - id
+                            retriever:
+                              type: SimpleRetriever
+                              requester:
+                                type: HttpRequester
+                                url_base: https://api.iterable.com/api/
+                                path: lists
+                                http_method: GET
+                                request_parameters: {}
+                                request_headers: {}
+                                authenticator:
+                                  type: ApiKeyAuthenticator
+                                  api_token: "{{ config['api_key'] }}"
+                                  inject_into:
+                                    type: RequestOption
+                                    field_name: Api-Key
+                                    inject_into: header
+                                request_body_json: {}
+                              record_selector:
+                                type: RecordSelector
+                                extractor:
+                                  type: DpathExtractor
+                                  field_path:
+                                    - lists
+                              paginator:
+                                type: NoPagination
+                              partition_router: []
+                primary_key:
+                  - id
+                transformations:
+                  - type: AddFields
+                    fields:
+                      - path:
+                          - list_id
+                        value: "{{ stream_slice.list_id }}"
+version: 0.65.0
+metadata:
+  autoImportSchema:
+    users: false
+    lists: false
+    channels: false
+    metadata: false
+    campaigns: false
+    message_types: false

source_iterable/source.py

@@ -2,20 +2,14 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from typing import Any, List, Mapping, Tuple
+from typing import Any, List, Mapping
 
-import requests.exceptions
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
-from source_iterable.utils import read_full_refresh
 
 from .streams import (
-    AccessCheck,
-    Campaigns,
     CampaignsMetrics,
-    Channels,
     CustomEvent,
     EmailBounce,
     EmailClick,
@@ -25,7 +19,6 @@ from .streams import (
     EmailSendSkip,
     EmailSubscribe,
     EmailUnsubscribe,
-    Events,
     HostedUnsubscribeClick,
     InAppClick,
     InAppClose,
@@ -36,10 +29,6 @@ from .streams import (
     InAppSendSkip,
     InboxMessageImpression,
     InboxSession,
-    Lists,
-    ListUsers,
-    MessageTypes,
-    Metadata,
     Purchase,
     PushBounce,
     PushOpen,
@@ -53,101 +42,70 @@ from .streams import (
     SmsSendSkip,
     SmsUsageInfo,
     Templates,
-    Users,
     WebPushClick,
     WebPushSend,
     WebPushSendSkip,
 )
 
+"""
+This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
+source connector.
 
-class SourceIterable(AbstractSource):
-    """
-    Note: there are some redundant endpoints
-    (e.g. [`export/userEvents`](https://api.iterable.com/api/docs#export_exportUserEvents)
-    and [`events/{email}`](https://api.iterable.com/api/docs#events_User_events)).
-    In this case it's better to use the one which takes params as a query param rather than as part of the url param.
-    """
+WARNING: Do not modify this file.
+"""
 
-    def check_connection(self, logger, config) -> Tuple[bool, any]:
-        try:
-            authenticator = TokenAuthenticator(token=config["api_key"], auth_header="Api-Key", auth_method="")
-            list_gen = Lists(authenticator=authenticator).read_records(sync_mode=SyncMode.full_refresh)
-            next(list_gen)
-            return True, None
-        except Exception as e:
-            return False, f"Unable to connect to Iterable API with the provided credentials - {e}"
+# Declarative Source
+class SourceIterable(YamlDeclarativeSource):
+    def __init__(self):
+        super().__init__(**{"path_to_yaml": "manifest.yaml"})
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        def all_streams_accessible():
-            access_check_stream = AccessCheck(authenticator=authenticator)
-            try:
-                next(read_full_refresh(access_check_stream), None)
-            except requests.exceptions.RequestException as e:
-                if e.response.status_code == requests.codes.UNAUTHORIZED:
-                    return False
-                raise
-            return True
+        streams = super().streams(config=config)
 
         authenticator = TokenAuthenticator(token=config["api_key"], auth_header="Api-Key", auth_method="")
         # end date is provided for integration tests only
         start_date, end_date = config["start_date"], config.get("end_date")
         date_range = {"start_date": start_date, "end_date": end_date}
-        streams = [
-            Campaigns(authenticator=authenticator),
-            CampaignsMetrics(authenticator=authenticator, **date_range),
-            Channels(authenticator=authenticator),
-            Lists(authenticator=authenticator),
-            MessageTypes(authenticator=authenticator),
-            Metadata(authenticator=authenticator),
-            Templates(authenticator=authenticator, **date_range),
-        ]
-        # Iterable supports two types of Server-side api keys:
-        # - read only
-        # - server side
-        # The first one has a limited set of supported APIs, so others are filtered out here.
-        # A simple check is done - a read operation on a stream that can be accessed only via a Server side API key.
-        # If read is successful - other streams should be supported as well.
-        # More on this - https://support.iterable.com/hc/en-us/articles/360043464871-API-Keys-
-        if all_streams_accessible():
-            streams.extend(
-                [
-                    Users(authenticator=authenticator, **date_range),
-                    ListUsers(authenticator=authenticator),
-                    EmailBounce(authenticator=authenticator, **date_range),
-                    EmailClick(authenticator=authenticator, **date_range),
-                    EmailComplaint(authenticator=authenticator, **date_range),
-                    EmailOpen(authenticator=authenticator, **date_range),
-                    EmailSend(authenticator=authenticator, **date_range),
-                    EmailSendSkip(authenticator=authenticator, **date_range),
-                    EmailSubscribe(authenticator=authenticator, **date_range),
-                    EmailUnsubscribe(authenticator=authenticator, **date_range),
-                    PushSend(authenticator=authenticator, **date_range),
-                    PushSendSkip(authenticator=authenticator, **date_range),
-                    PushOpen(authenticator=authenticator, **date_range),
-                    PushUninstall(authenticator=authenticator, **date_range),
-                    PushBounce(authenticator=authenticator, **date_range),
-                    WebPushSend(authenticator=authenticator, **date_range),
-                    WebPushClick(authenticator=authenticator, **date_range),
-                    WebPushSendSkip(authenticator=authenticator, **date_range),
-                    InAppSend(authenticator=authenticator, **date_range),
-                    InAppOpen(authenticator=authenticator, **date_range),
-                    InAppClick(authenticator=authenticator, **date_range),
-                    InAppClose(authenticator=authenticator, **date_range),
-                    InAppDelete(authenticator=authenticator, **date_range),
-                    InAppDelivery(authenticator=authenticator, **date_range),
-                    InAppSendSkip(authenticator=authenticator, **date_range),
-                    InboxSession(authenticator=authenticator, **date_range),
-                    InboxMessageImpression(authenticator=authenticator, **date_range),
-                    SmsSend(authenticator=authenticator, **date_range),
-                    SmsBounce(authenticator=authenticator, **date_range),
-                    SmsClick(authenticator=authenticator, **date_range),
-                    SmsReceived(authenticator=authenticator, **date_range),
-                    SmsSendSkip(authenticator=authenticator, **date_range),
-                    SmsUsageInfo(authenticator=authenticator, **date_range),
-                    Purchase(authenticator=authenticator, **date_range),
-                    CustomEvent(authenticator=authenticator, **date_range),
-                    HostedUnsubscribeClick(authenticator=authenticator, **date_range),
-                    Events(authenticator=authenticator),
-                ]
-            )
+
+        # TODO: migrate streams below to low code as slicer logic will be migrated to generator based
+        streams.extend(
+            [
+                CampaignsMetrics(authenticator=authenticator, **date_range),
+                Templates(authenticator=authenticator, **date_range),
+                EmailBounce(authenticator=authenticator, **date_range),
+                EmailClick(authenticator=authenticator, **date_range),
+                EmailComplaint(authenticator=authenticator, **date_range),
+                EmailOpen(authenticator=authenticator, **date_range),
+                EmailSend(authenticator=authenticator, **date_range),
+                EmailSendSkip(authenticator=authenticator, **date_range),
+                EmailSubscribe(authenticator=authenticator, **date_range),
+                EmailUnsubscribe(authenticator=authenticator, **date_range),
+                PushSend(authenticator=authenticator, **date_range),
+                PushSendSkip(authenticator=authenticator, **date_range),
+                PushOpen(authenticator=authenticator, **date_range),
+                PushUninstall(authenticator=authenticator, **date_range),
+                PushBounce(authenticator=authenticator, **date_range),
+                WebPushSend(authenticator=authenticator, **date_range),
+                WebPushClick(authenticator=authenticator, **date_range),
+                WebPushSendSkip(authenticator=authenticator, **date_range),
+                InAppSend(authenticator=authenticator, **date_range),
+                InAppOpen(authenticator=authenticator, **date_range),
+                InAppClick(authenticator=authenticator, **date_range),
+                InAppClose(authenticator=authenticator, **date_range),
+                InAppDelete(authenticator=authenticator, **date_range),
+                InAppDelivery(authenticator=authenticator, **date_range),
+                InAppSendSkip(authenticator=authenticator, **date_range),
+                InboxSession(authenticator=authenticator, **date_range),
+                InboxMessageImpression(authenticator=authenticator, **date_range),
+                SmsSend(authenticator=authenticator, **date_range),
+                SmsBounce(authenticator=authenticator, **date_range),
+                SmsClick(authenticator=authenticator, **date_range),
+                SmsReceived(authenticator=authenticator, **date_range),
+                SmsSendSkip(authenticator=authenticator, **date_range),
+                SmsUsageInfo(authenticator=authenticator, **date_range),
+                Purchase(authenticator=authenticator, **date_range),
+                CustomEvent(authenticator=authenticator, **date_range),
+                HostedUnsubscribeClick(authenticator=authenticator, **date_range),
+            ]
+        )
         return streams

source_iterable/spec.json

@@ -1,5 +1,4 @@
 {
-  "documentationUrl": "https://docs.airbyte.com/integrations/sources/iterable",
   "connectionSpecification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Iterable Spec",
@@ -10,14 +9,14 @@
       "api_key": {
         "type": "string",
         "title": "API Key",
-        "description": "Iterable API Key. See the <a href=\"https://docs.airbyte.com/integrations/sources/iterable\">docs</a> for more information on how to obtain this key.",
+        "description": "Iterable API Key. See the <a href=\\\"https://docs.airbyte.com/integrations/sources/iterable\\\">docs</a>  for more information on how to obtain this key.",
         "airbyte_secret": true,
         "order": 0
       },
       "start_date": {
         "type": "string",
         "title": "Start Date",
-        "description": "The date from which you'd like to replicate data for Iterable, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.",
+        "description": "The date from which you'd like to replicate data for Iterable, in the format YYYY-MM-DDT00:00:00Z.  All data generated after this date will be replicated.",
         "examples": ["2021-04-01T00:00:00Z"],
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
         "order": 1,

source_iterable/streams.py

@@ -4,7 +4,6 @@
 
 import csv
 import json
-import urllib.parse as urlparse
 from abc import ABC, abstractmethod
 from io import StringIO
 from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union
@@ -66,11 +65,6 @@ class IterableStream(HttpStream, ABC):
         """
         return None
 
-    def check_unauthorized_key(self, response: requests.Response) -> bool:
-        if response.status_code == codes.UNAUTHORIZED:
-            self.logger.warning(f"Provided API Key has not sufficient permissions to read from stream: {self.data_field}")
-            return True
-
     def check_generic_error(self, response: requests.Response) -> bool:
         """
         https://github.com/airbytehq/oncall/issues/1592#issuecomment-1499109251
@@ -129,9 +123,6 @@ class IterableStream(HttpStream, ABC):
             yield from super().read_records(sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state)
         except (HTTPError, UserDefinedBackoffException, DefaultBackoffException) as e:
             response = e.response
-            if self.check_unauthorized_key(response):
-                self.ignore_further_slices = True
-                return
             if self.check_generic_error(response):
                 return
             raise e
@@ -342,42 +333,6 @@ class IterableExportEventsStreamAdjustableRange(IterableExportStreamAdjustableRa
         return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("events")
 
 
-class Lists(IterableStream):
-    data_field = "lists"
-
-    def path(self, **kwargs) -> str:
-        return "lists"
-
-
-class ListUsers(IterableStream):
-    primary_key = "listId"
-    data_field = "getUsers"
-    name = "list_users"
-    # enable caching, because this stream used by other ones
-    use_cache = True
-
-    def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str:
-        return f"lists/{self.data_field}?listId={stream_slice['list_id']}"
-
-    def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
-        lists = Lists(authenticator=self._cred)
-        for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh)):
-            yield {"list_id": list_record["id"]}
-
-    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        list_id = self._get_list_id(response.url)
-        for user in response.iter_lines():
-            yield {"email": user.decode(), "listId": list_id}
-
-    @staticmethod
-    def _get_list_id(url: str) -> int:
-        parsed_url = urlparse.urlparse(url)
-        for q in parsed_url.query.split("&"):
-            key, value = q.split("=")
-            if key == "listId":
-                return int(value)
-
-
 class Campaigns(IterableStream):
     data_field = "campaigns"
 
@@ -465,71 +420,6 @@ class CampaignsMetrics(IterableStream):
         return result
 
 
-class Channels(IterableStream):
-    data_field = "channels"
-
-    def path(self, **kwargs) -> str:
-        return "channels"
-
-
-class MessageTypes(IterableStream):
-    data_field = "messageTypes"
-    name = "message_types"
-
-    def path(self, **kwargs) -> str:
-        return "messageTypes"
-
-
-class Metadata(IterableStream):
-    primary_key = None
-    data_field = "results"
-
-    def path(self, **kwargs) -> str:
-        return "metadata"
-
-
-class Events(IterableStream):
-    """
-    https://api.iterable.com/api/docs#export_exportUserEvents
-    """
-
-    primary_key = None
-    data_field = "events"
-    common_fields = ("itblInternal", "_type", "createdAt", "email")
-
-    def path(self, **kwargs) -> str:
-        return "export/userEvents"
-
-    def request_params(self, stream_slice: Optional[Mapping[str, Any]], **kwargs) -> MutableMapping[str, Any]:
-        params = super().request_params(**kwargs)
-        params.update({"email": stream_slice["email"], "includeCustomEvents": "true"})
-
-        return params
-
-    def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
-        lists = ListUsers(authenticator=self._cred)
-        stream_slices = lists.stream_slices()
-
-        for stream_slice in stream_slices:
-            for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh), stream_slice=stream_slice):
-                yield {"email": list_record["email"]}
-
-    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        """
-        Parse jsonl response body.
-        Put common event fields at the top level.
-        Put the rest of the fields in the `data` subobject.
-        """
-        jsonl_records = StringIO(response.text)
-        for record in jsonl_records:
-            record_dict = json.loads(record)
-            record_dict_common_fields = {}
-            for field in self.common_fields:
-                record_dict_common_fields[field] = record_dict.pop(field, None)
-
-            yield {**record_dict_common_fields, "data": record_dict}
-
-
 class EmailBounce(IterableExportStreamAdjustableRange):
     data_field = "emailBounce"
 
@@ -687,16 +577,3 @@ class Templates(IterableExportStreamRanged):
         for record in records:
             record[self.cursor_field] = self._field_to_datetime(record[self.cursor_field])
             yield record
-
-
-class Users(IterableExportStreamRanged):
-    data_field = "user"
-    cursor_field = "profileUpdatedAt"
-
-
-class AccessCheck(ListUsers):
-    # since 401 error is failed silently in all the streams,
-    # we need another class to distinguish an empty stream from 401 response
-    def check_unauthorized_key(self, response: requests.Response) -> bool:
-        # this allows not retrying 401 and raising the error upstream
-        return response.status_code != codes.UNAUTHORIZED

source_iterable/utils.py

@@ -24,10 +24,3 @@ def dateutil_parse(text):
         dt.microsecond,
         tz=dt.tzinfo or pendulum.tz.UTC,
     )
-
-
-def read_full_refresh(stream_instance: Stream):
-    slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh)
-    for _slice in slices:
-        for record in stream_instance.read_records(stream_slice=_slice, sync_mode=SyncMode.full_refresh):
-            yield record