airbyte-source-iterable 0.5.11__py3-none-any.whl → 0.6.1__py3-none-any.whl

{airbyte_source_iterable-0.5.11.dist-info → airbyte_source_iterable-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-iterable
-Version: 0.5.11
+Version: 0.6.1
 Summary: Source implementation for Iterable.
 Home-page: https://airbyte.com
 License: MIT
@@ -12,10 +12,9 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (==0.90.0)
+Requires-Dist: airbyte-cdk (>=3,<4)
 Requires-Dist: pendulum (==2.1.2)
 Requires-Dist: python-dateutil (==2.8.2)
-Requires-Dist: requests (==2.31.0)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/iterable
 Project-URL: Repository, https://github.com/airbytehq/airbyte
 Description-Content-Type: text/markdown

{airbyte_source_iterable-0.5.11.dist-info → airbyte_source_iterable-0.6.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
 source_iterable/__init__.py,sha256=8WKQT800ggxG1vGPoA_ZHUkozwEM7qMhGMhzEPCdA4o,65
-source_iterable/components.py,sha256=dHp23THc43TmogP_tCxcv12mAgZSRs5fKir2ckr4hHI,1374
-source_iterable/manifest.yaml,sha256=uO7z-bIU6CtGms6IsLwjyZ-cb3tVY-vLUMLOZGkZDts,13996
+source_iterable/components.py,sha256=IuLcldPEti2hc8v4-KyTCLFla0pyuK0EBa7liHUS2wk,805
+source_iterable/manifest.yaml,sha256=BL0mAODdqpiRycEG5igczmkCA7y2VhDnqCAHrsPbUc4,14136
 source_iterable/run.py,sha256=-Bvz772Z7iJWExDoJS7chxv725hG4e9e5Engrtp66iE,236
 source_iterable/schemas/campaigns.json,sha256=FONoPuz_4aRat8OZzhj7BcHFAqetpBYUDmOZfroCu7I,2494
 source_iterable/schemas/campaigns_metrics.json,sha256=Aa6pNdVxHOG-LSxlLzhxFY8uIdfrPBTgX9QyWrLJ4Fs,168
@@ -23,9 +23,9 @@ source_iterable/schemas/users.json,sha256=CHqA30cnsTXfI-M0n54NwVaaZEO12tt_IgQ9yP
 source_iterable/slice_generators.py,sha256=neRuWD52Xnr84KvWQ0kjaErR44sOKjxIyLCNeofohzE,6482
 source_iterable/source.py,sha256=AhZQ4L54VTo6jf2QltvFCqDNgqaPSEKCfR0GInCLEEc,4537
 source_iterable/spec.json,sha256=y5_YFTkPPtxjvzGtaZZql90Tc37fyZZpQKXFGPL_cYE,1009
-source_iterable/streams.py,sha256=r6YZlquKUQCyoKlwdIOVMUCTCBLiQPS4POJKmuZD61s,19913
-source_iterable/utils.py,sha256=2oM8AjBZXs9nTG_PhSWmxBWEmp-w-tWFaxQQvAfUuMM,649
-airbyte_source_iterable-0.5.11.dist-info/METADATA,sha256=YD5fm3-HhbbWhuLoFG-RPNk4dbQ3_Me6NZURgNTxFwE,5368
-airbyte_source_iterable-0.5.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-airbyte_source_iterable-0.5.11.dist-info/entry_points.txt,sha256=pjzrNtsOX3jW37IK0U5pSmLiraiuLTPr5aB5-hBXpAo,59
-airbyte_source_iterable-0.5.11.dist-info/RECORD,,
+source_iterable/streams.py,sha256=wig4zwcSmEIgayutBmrPwhHv4I1ZAuEDLGEkt479NAk,19940
+source_iterable/utils.py,sha256=08uzNpJGDd3uogybhfAtrqBz4lFsldURWyt5OFpz79s,562
+airbyte_source_iterable-0.6.1.dist-info/METADATA,sha256=kpUjHEV8-tTTsnov1wKp_o9kWQI9MjhDhdIougpotms,5330
+airbyte_source_iterable-0.6.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+airbyte_source_iterable-0.6.1.dist-info/entry_points.txt,sha256=pjzrNtsOX3jW37IK0U5pSmLiraiuLTPr5aB5-hBXpAo,59
+airbyte_source_iterable-0.6.1.dist-info/RECORD,,

source_iterable/components.py

@@ -2,40 +2,21 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-import json
 from dataclasses import dataclass
-from io import StringIO
+from typing import Any, Iterable, Mapping
 
 import requests
 from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
-from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState
-
-
-@dataclass
-class XJsonRecordExtractor(DpathExtractor):
-    def extract_records(self, response: requests.Response) -> list[Record]:
-        return [json.loads(record) for record in response.iter_lines()]
-
-
-@dataclass
-class ListUsersRecordExtractor(DpathExtractor):
-    def extract_records(self, response: requests.Response) -> list[Record]:
-        return [{"email": record.decode()} for record in response.iter_lines()]
 
 
 @dataclass
 class EventsRecordExtractor(DpathExtractor):
     common_fields = ("itblInternal", "_type", "createdAt", "email")
 
-    def extract_records(self, response: requests.Response) -> list[Record]:
-        jsonl_records = StringIO(response.text)
-        records = []
-        for record in jsonl_records:
-            record_dict = json.loads(record)
+    def extract_records(self, response: requests.Response) -> Iterable[Mapping[str, Any]]:
+        jsonl_records = super().extract_records(response=response)
+        for record_dict in jsonl_records:
             record_dict_common_fields = {}
             for field in self.common_fields:
                 record_dict_common_fields[field] = record_dict.pop(field, None)
-
-            records.append({**record_dict_common_fields, "data": record_dict})
-
-        return records
+            yield {**record_dict_common_fields, "data": record_dict}

source_iterable/manifest.yaml

@@ -67,6 +67,8 @@ streams:
     type: DeclarativeStream
     retriever:
       type: SimpleRetriever
+      decoder:
+        type: IterableDecoder
       paginator:
         type: NoPagination
       requester:
@@ -87,9 +89,8 @@ streams:
       record_selector:
         type: RecordSelector
         extractor:
-          class_name: source_iterable.components.ListUsersRecordExtractor
-          field_path:
-            - getUsers
+          type: DpathExtractor
+          field_path: []
       partition_router:
         - type: SubstreamPartitionRouter
           parent_stream_configs:
@@ -139,6 +140,12 @@ streams:
           - path:
               - listId
             value: "{{ stream_slice.list_id }}"
+          - path:
+              - email
+            value: "{{ record.record }}"
+      - type: RemoveFields
+        field_pointers:
+          - - record
   - name: campaigns
     type: DeclarativeStream
     retriever:
@@ -262,6 +269,8 @@ streams:
     type: DeclarativeStream
     retriever:
       type: SimpleRetriever
+      decoder:
+        type: JsonlDecoder
       paginator:
         type: NoPagination
       requester:
@@ -284,9 +293,8 @@ streams:
       record_selector:
         type: RecordSelector
         extractor:
-          class_name: source_iterable.components.XJsonRecordExtractor
-          field_path:
-            - users
+          type: DpathExtractor
+          field_path: []
       partition_router: []
     primary_key: []
     incremental_sync:
@@ -318,6 +326,8 @@ streams:
     primary_key: []
     retriever:
       type: SimpleRetriever
+      decoder:
+        type: JsonlDecoder
       requester:
         type: HttpRequester
         url_base: https://api.iterable.com/api/
@@ -338,15 +348,14 @@ streams:
         type: RecordSelector
         extractor:
           class_name: source_iterable.components.EventsRecordExtractor
-          field_path:
-            - events
+          field_path: []
       paginator:
         type: NoPagination
       partition_router:
         - type: SubstreamPartitionRouter
           parent_stream_configs:
             - type: ParentStreamConfig
-              parent_key: email
+              parent_key: record
               request_option:
                 inject_into: request_parameter
                 type: RequestOption
@@ -357,6 +366,8 @@ streams:
                 type: DeclarativeStream
                 retriever:
                   type: SimpleRetriever
+                  decoder:
+                    type: IterableDecoder
                   paginator:
                     type: NoPagination
                   requester:
@@ -377,9 +388,8 @@ streams:
                   record_selector:
                     type: RecordSelector
                     extractor:
-                      class_name: source_iterable.components.ListUsersRecordExtractor
-                      field_path:
-                        - getUsers
+                      type: DpathExtractor
+                      field_path: []
                   partition_router:
                     - type: SubstreamPartitionRouter
                       parent_stream_configs:

source_iterable/streams.py

@@ -17,7 +17,7 @@ from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException
 from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
 from pendulum.datetime import DateTime
-from requests import HTTPError, codes
+from requests import HTTPError
 from requests.exceptions import ChunkedEncodingError
 from source_iterable.slice_generators import AdjustableSliceGenerator, RangeSliceGenerator, StreamSlice
 from source_iterable.utils import dateutil_parse
@@ -106,7 +106,7 @@ class IterableStream(HttpStream, ABC):
             if self._slice_retry < 3:
                 return True
             return False
-        return super().should_retry(response)
+        return response.status_code == 429 or 500 <= response.status_code < 600
 
     def read_records(
         self,

source_iterable/utils.py

@@ -4,8 +4,6 @@
 
 import dateutil.parser
 import pendulum
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources.streams import Stream
 
 
 def dateutil_parse(text):