airbyte-cdk 0.55.5__py3-none-any.whl → 0.56.1__py3-none-any.whl

airbyte_cdk/sources/abstract_source.py

@@ -101,6 +101,9 @@ class AbstractSource(Source, ABC):
         stream_instances = {s.name: s for s in self.streams(config)}
         state_manager = ConnectorStateManager(stream_instance_map=stream_instances, state=state)
         self._stream_to_instance_map = stream_instances
+
+        stream_name_to_exception: MutableMapping[str, AirbyteTracedException] = {}
+
         with create_timer(self.name) as timer:
             for configured_stream in catalog.streams:
                 stream_instance = stream_instances.get(configured_stream.stream.name)
@@ -131,7 +134,10 @@ class AbstractSource(Source, ABC):
                     yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.COMPLETE)
                 except AirbyteTracedException as e:
                     yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.INCOMPLETE)
-                    raise e
+                    if self.continue_sync_on_stream_failure:
+                        stream_name_to_exception[stream_instance.name] = e
+                    else:
+                        raise e
                 except Exception as e:
                     yield from self._emit_queued_messages()
                     logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}")
@@ -146,6 +152,8 @@ class AbstractSource(Source, ABC):
                     logger.info(f"Finished syncing {configured_stream.stream.name}")
                     logger.info(timer.report())
 
+        if self.continue_sync_on_stream_failure and len(stream_name_to_exception) > 0:
+            raise AirbyteTracedException(message=self._generate_failed_streams_error_message(stream_name_to_exception))
         logger.info(f"Finished syncing {self.name}")
 
     @property
@@ -272,3 +280,19 @@ class AbstractSource(Source, ABC):
     @property
     def message_repository(self) -> Union[None, MessageRepository]:
         return _default_message_repository
+
+    @property
+    def continue_sync_on_stream_failure(self) -> bool:
+        """
+        WARNING: This function is in-development which means it is subject to change. Use at your own risk.
+
+        By default, a source should raise an exception and stop the sync when it encounters an error while syncing a stream. This
+        method can be overridden on a per-source basis so that a source will continue syncing streams other streams even if an
+        exception is raised for a stream.
+        """
+        return False
+
+    @staticmethod
+    def _generate_failed_streams_error_message(stream_failures: Mapping[str, AirbyteTracedException]) -> str:
+        failures = ", ".join([f"{stream}: {exception.__repr__()}" for stream, exception in stream_failures.items()])
+        return f"During the sync, the following streams did not sync successfully: {failures}"

{airbyte_cdk-0.55.5.dist-info → airbyte_cdk-0.56.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.55.5
+Version: 0.56.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.55.5.dist-info → airbyte_cdk-0.56.1.dist-info}/RECORD

@@ -24,7 +24,7 @@ airbyte_cdk/models/__init__.py,sha256=Kg8YHBqUsNWHlAw-u3ZGdG4dxLh7qBlHhqMRfamNCR
 airbyte_cdk/models/airbyte_protocol.py,sha256=DoJvnmGM3xMAZFTwA6_RGMiKSFqfE3ib_Ru0KJ65Ag4,100
 airbyte_cdk/models/well_known_types.py,sha256=KKfNbow2gdLoC1Z4hcXy_JR8m_acsB2ol7gQuEgjobw,117
 airbyte_cdk/sources/__init__.py,sha256=Ov7Uf03KPSZUmMZqZfUAK3tQwsdKjDQUDvTb-H0JyfA,1141
-airbyte_cdk/sources/abstract_source.py,sha256=tlXXwCRBisbOu7pA2NYtiRolhHSCaKlXcET6dny5V9o,13043
+airbyte_cdk/sources/abstract_source.py,sha256=w-f9Z2sp8xAk4MHVkr-K4tBP1NMxBIGRCd9NsHm2xBE,14365
 airbyte_cdk/sources/config.py,sha256=PYsY7y2u3EUwxLiEb96JnuKwH_E8CuxKggsRO2ZPSRc,856
 airbyte_cdk/sources/connector_state_manager.py,sha256=wsmUgII398MazCTKxwLBLzeiU6Z-tMTrKX882EEy-YE,10904
 airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
@@ -269,7 +269,7 @@ unit_tests/singer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 unit_tests/singer/test_singer_helpers.py,sha256=pZV6VxJuK-3-FICNGmoGbokrA_zkaFZEd4rYZCVpSRU,1762
 unit_tests/singer/test_singer_source.py,sha256=edN_kv7dnYAdBveWdUYOs74ak0dK6p8uaX225h_ZILA,4442
 unit_tests/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unit_tests/sources/test_abstract_source.py,sha256=V7zSpOk-jqfOz8FtnImAo_zDe-Q2TjPqD_l_T0QaiDw,48179
+unit_tests/sources/test_abstract_source.py,sha256=_wFBhJ2dsxWpYqw_QSGL7WwJyqVvXLQOMN3GrD5nC4A,52519
 unit_tests/sources/test_concurrent_source.py,sha256=NT4K0z-oz2OZBHE9xNQT0KUdI2wJ-5vNWLUHZlIYKKU,3552
 unit_tests/sources/test_config.py,sha256=lxjeaf48pOMF4Pf3-Z1ux_tHTyjRFCdG_hpnxw3e7uQ,2839
 unit_tests/sources/test_connector_state_manager.py,sha256=ynFxA63Cxe6t-wMMh9C6ByTlMAuk8W7H2FikDhnUEQ0,24264
@@ -437,8 +437,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
 unit_tests/utils/test_stream_status_utils.py,sha256=Xr8MZ2HWgTVIyMbywDvuYkRaUF4RZLQOT8-JjvcfR24,2970
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.55.5.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.55.5.dist-info/METADATA,sha256=UGBscoQLxD10YK6vK2sziE_7IkU_JY03BSihYZcEJv0,11983
-airbyte_cdk-0.55.5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-airbyte_cdk-0.55.5.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.55.5.dist-info/RECORD,,
+airbyte_cdk-0.56.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.56.1.dist-info/METADATA,sha256=9SrM3BpVcDEm06b_24RA58-vED_17w1-dZbGme1dt3Q,11983
+airbyte_cdk-0.56.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+airbyte_cdk-0.56.1.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.56.1.dist-info/RECORD,,

unit_tests/sources/test_abstract_source.py

@@ -111,6 +111,14 @@ class MockStreamOverridesStateMethod(Stream, IncrementalMixin):
         self._cursor_value = value.get(self.cursor_field, self.start_date)
 
 
+class StreamRaisesException(Stream):
+    name = "lamentations"
+    primary_key = None
+
+    def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]:
+        raise AirbyteTracedException(message="I was born only to crash like Icarus")
+
+
 MESSAGE_FROM_REPOSITORY = Mock()
 
 
@@ -1158,3 +1166,97 @@ def test_checkpoint_state_from_stream_instance():
     assert actual_message == _as_state(
         {"teams": {"updated_at": "2022-09-11"}, "managers": {"updated": "expected_here"}}, "managers", {"updated": "expected_here"}
     )
+
+
+def test_continue_sync_with_failed_streams(mocker):
+    """
+    Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure enabled continues
+    syncing even when one stream fails with an error.
+    """
+    stream_output = [{"k1": "v1"}, {"k2": "v2"}]
+    s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
+    s2 = StreamRaisesException()
+    s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
+
+    mocker.patch.object(MockStream, "get_json_schema", return_value={})
+    mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
+
+    src = MockSource(streams=[s1, s2, s3])
+    mocker.patch.object(MockSource, "continue_sync_on_stream_failure", return_value=True)
+    catalog = ConfiguredAirbyteCatalog(
+        streams=[
+            _configured_stream(s1, SyncMode.full_refresh),
+            _configured_stream(s2, SyncMode.full_refresh),
+            _configured_stream(s3, SyncMode.full_refresh),
+        ]
+    )
+
+    expected = _fix_emitted_at(
+        [
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
+            *_as_records("s1", stream_output),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
+            _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
+            _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
+            _as_stream_status("s3", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s3", AirbyteStreamStatus.RUNNING),
+            *_as_records("s3", stream_output),
+            _as_stream_status("s3", AirbyteStreamStatus.COMPLETE),
+        ]
+    )
+
+    messages = []
+    with pytest.raises(AirbyteTracedException) as exc:
+        # We can't use list comprehension or list() here because we are still raising a final exception for the
+        # failed streams and that disrupts parsing the generator into the messages emitted before
+        for message in src.read(logger, {}, catalog):
+            messages.append(message)
+
+    messages = _fix_emitted_at(messages)
+    assert expected == messages
+    assert "lamentations" in exc.value.message
+
+
+def test_stop_sync_with_failed_streams(mocker):
+    """
+    Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure disabled stops
+    syncing once a stream fails with an error.
+    """
+    stream_output = [{"k1": "v1"}, {"k2": "v2"}]
+    s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
+    s2 = StreamRaisesException()
+    s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
+
+    mocker.patch.object(MockStream, "get_json_schema", return_value={})
+    mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
+
+    src = MockSource(streams=[s1, s2, s3])
+    catalog = ConfiguredAirbyteCatalog(
+        streams=[
+            _configured_stream(s1, SyncMode.full_refresh),
+            _configured_stream(s2, SyncMode.full_refresh),
+            _configured_stream(s3, SyncMode.full_refresh),
+        ]
+    )
+
+    expected = _fix_emitted_at(
+        [
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
+            *_as_records("s1", stream_output),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
+            _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
+            _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
+        ]
+    )
+
+    messages = []
+    with pytest.raises(AirbyteTracedException):
+        # We can't use list comprehension or list() here because we are still raising a final exception for the
+        # failed streams and that disrupts parsing the generator into the messages emitted before
+        for message in src.read(logger, {}, catalog):
+            messages.append(message)
+
+    messages = _fix_emitted_at(messages)
+    assert expected == messages