airbyte-source-iterable 0.5.10__tar.gz → 0.6.0__tar.gz

{airbyte_source_iterable-0.5.10 → airbyte_source_iterable-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-iterable
-Version: 0.5.10
+Version: 0.6.0
 Summary: Source implementation for Iterable.
 Home-page: https://airbyte.com
 License: MIT
@@ -12,10 +12,9 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (==0.90.0)
+Requires-Dist: airbyte-cdk (>=3,<4)
 Requires-Dist: pendulum (==2.1.2)
 Requires-Dist: python-dateutil (==2.8.2)
-Requires-Dist: requests (==2.31.0)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/iterable
 Project-URL: Repository, https://github.com/airbytehq/airbyte
 Description-Content-Type: text/markdown

{airbyte_source_iterable-0.5.10 → airbyte_source_iterable-0.6.0}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "0.5.10"
+version = "0.6.0"
 name = "airbyte-source-iterable"
 description = "Source implementation for Iterable."
 authors = [
@@ -23,16 +23,15 @@ packages = [
 [tool.poetry.dependencies]
 python = "^3.9,<3.12"
 pendulum = "==2.1.2"
-airbyte-cdk = "0.90.0"
-requests = "==2.31.0"
+airbyte-cdk = "^3"
 python-dateutil = "==2.8.2"
 
 [tool.poetry.scripts]
 source-iterable = "source_iterable.run:run"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^6.1"
+pytest = "^6.2"
 pytest-mock = "^3.6.1"
-freezegun = "==1.1.0"
+freezegun = "==1.5.1"
 requests-mock = "^1.9.3"
 responses = "==0.23.1"

airbyte_source_iterable-0.6.0/source_iterable/components.py

@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import dataclass
+from typing import Any, Iterable, Mapping
+
+import requests
+from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
+
+
+@dataclass
+class EventsRecordExtractor(DpathExtractor):
+    common_fields = ("itblInternal", "_type", "createdAt", "email")
+
+    def extract_records(self, response: requests.Response) -> Iterable[Mapping[str, Any]]:
+        jsonl_records = super().extract_records(response=response)
+        for record_dict in jsonl_records:
+            record_dict_common_fields = {}
+            for field in self.common_fields:
+                record_dict_common_fields[field] = record_dict.pop(field, None)
+            yield {**record_dict_common_fields, "data": record_dict}

{airbyte_source_iterable-0.5.10 → airbyte_source_iterable-0.6.0}/source_iterable/manifest.yaml

@@ -67,6 +67,8 @@ streams:
     type: DeclarativeStream
     retriever:
       type: SimpleRetriever
+      decoder:
+        type: IterableDecoder
       paginator:
         type: NoPagination
       requester:
@@ -87,9 +89,8 @@ streams:
       record_selector:
         type: RecordSelector
         extractor:
-          class_name: source_iterable.components.ListUsersRecordExtractor
-          field_path:
-            - getUsers
+          type: DpathExtractor
+          field_path: []
       partition_router:
         - type: SubstreamPartitionRouter
           parent_stream_configs:
@@ -139,6 +140,12 @@ streams:
           - path:
               - listId
             value: "{{ stream_slice.list_id }}"
+          - path:
+              - email
+            value: "{{ record.record }}"
+      - type: RemoveFields
+        field_pointers:
+          - - record
   - name: campaigns
     type: DeclarativeStream
     retriever:
@@ -262,6 +269,8 @@ streams:
     type: DeclarativeStream
     retriever:
       type: SimpleRetriever
+      decoder:
+        type: JsonlDecoder
       paginator:
         type: NoPagination
       requester:
@@ -284,9 +293,8 @@ streams:
       record_selector:
         type: RecordSelector
         extractor:
-          class_name: source_iterable.components.XJsonRecordExtractor
-          field_path:
-            - users
+          type: DpathExtractor
+          field_path: []
       partition_router: []
     primary_key: []
     incremental_sync:
@@ -318,6 +326,8 @@ streams:
     primary_key: []
     retriever:
       type: SimpleRetriever
+      decoder:
+        type: JsonlDecoder
       requester:
         type: HttpRequester
         url_base: https://api.iterable.com/api/
@@ -338,15 +348,14 @@ streams:
         type: RecordSelector
         extractor:
           class_name: source_iterable.components.EventsRecordExtractor
-          field_path:
-            - events
+          field_path: []
       paginator:
         type: NoPagination
       partition_router:
         - type: SubstreamPartitionRouter
           parent_stream_configs:
             - type: ParentStreamConfig
-              parent_key: email
+              parent_key: record
               request_option:
                 inject_into: request_parameter
                 type: RequestOption
@@ -357,6 +366,8 @@ streams:
                 type: DeclarativeStream
                 retriever:
                   type: SimpleRetriever
+                  decoder:
+                    type: IterableDecoder
                   paginator:
                     type: NoPagination
                   requester:
@@ -377,9 +388,8 @@ streams:
                   record_selector:
                     type: RecordSelector
                     extractor:
-                      class_name: source_iterable.components.ListUsersRecordExtractor
-                      field_path:
-                        - getUsers
+                      type: DpathExtractor
+                      field_path: []
                   partition_router:
                     - type: SubstreamPartitionRouter
                       parent_stream_configs:

{airbyte_source_iterable-0.5.10 → airbyte_source_iterable-0.6.0}/source_iterable/streams.py

@@ -17,7 +17,7 @@ from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException
 from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
 from pendulum.datetime import DateTime
-from requests import HTTPError, codes
+from requests import HTTPError
 from requests.exceptions import ChunkedEncodingError
 from source_iterable.slice_generators import AdjustableSliceGenerator, RangeSliceGenerator, StreamSlice
 from source_iterable.utils import dateutil_parse
@@ -106,7 +106,7 @@ class IterableStream(HttpStream, ABC):
             if self._slice_retry < 3:
                 return True
             return False
-        return super().should_retry(response)
+        return response.status_code == 429 or 500 <= response.status_code < 600
 
     def read_records(
         self,

{airbyte_source_iterable-0.5.10 → airbyte_source_iterable-0.6.0}/source_iterable/utils.py

@@ -4,8 +4,6 @@
 
 import dateutil.parser
 import pendulum
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources.streams import Stream
 
 
 def dateutil_parse(text):

airbyte_source_iterable-0.5.10/source_iterable/components.py

@@ -1,41 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-from dataclasses import dataclass
-from io import StringIO
-
-import requests
-from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
-from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState
-
-
-@dataclass
-class XJsonRecordExtractor(DpathExtractor):
-    def extract_records(self, response: requests.Response) -> list[Record]:
-        return [json.loads(record) for record in response.iter_lines()]
-
-
-@dataclass
-class ListUsersRecordExtractor(DpathExtractor):
-    def extract_records(self, response: requests.Response) -> list[Record]:
-        return [{"email": record.decode()} for record in response.iter_lines()]
-
-
-@dataclass
-class EventsRecordExtractor(DpathExtractor):
-    common_fields = ("itblInternal", "_type", "createdAt", "email")
-
-    def extract_records(self, response: requests.Response) -> list[Record]:
-        jsonl_records = StringIO(response.text)
-        records = []
-        for record in jsonl_records:
-            record_dict = json.loads(record)
-            record_dict_common_fields = {}
-            for field in self.common_fields:
-                record_dict_common_fields[field] = record_dict.pop(field, None)
-
-            records.append({**record_dict_common_fields, "data": record_dict})
-
-        return records