airbyte-cdk 0.55.5__tar.gz → 0.56.1__tar.gz

{airbyte-cdk-0.55.5 → airbyte-cdk-0.56.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.55.5
+Version: 0.56.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.55.5 → airbyte-cdk-0.56.1}/airbyte_cdk/sources/abstract_source.py

@@ -101,6 +101,9 @@ class AbstractSource(Source, ABC):
         stream_instances = {s.name: s for s in self.streams(config)}
         state_manager = ConnectorStateManager(stream_instance_map=stream_instances, state=state)
         self._stream_to_instance_map = stream_instances
+
+        stream_name_to_exception: MutableMapping[str, AirbyteTracedException] = {}
+
         with create_timer(self.name) as timer:
             for configured_stream in catalog.streams:
                 stream_instance = stream_instances.get(configured_stream.stream.name)
@@ -131,7 +134,10 @@ class AbstractSource(Source, ABC):
                     yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.COMPLETE)
                 except AirbyteTracedException as e:
                     yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.INCOMPLETE)
-                    raise e
+                    if self.continue_sync_on_stream_failure:
+                        stream_name_to_exception[stream_instance.name] = e
+                    else:
+                        raise e
                 except Exception as e:
                     yield from self._emit_queued_messages()
                     logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}")
@@ -146,6 +152,8 @@ class AbstractSource(Source, ABC):
                     logger.info(f"Finished syncing {configured_stream.stream.name}")
                     logger.info(timer.report())
 
+        if self.continue_sync_on_stream_failure and len(stream_name_to_exception) > 0:
+            raise AirbyteTracedException(message=self._generate_failed_streams_error_message(stream_name_to_exception))
         logger.info(f"Finished syncing {self.name}")
 
     @property
@@ -272,3 +280,19 @@ class AbstractSource(Source, ABC):
     @property
     def message_repository(self) -> Union[None, MessageRepository]:
         return _default_message_repository
+
+    @property
+    def continue_sync_on_stream_failure(self) -> bool:
+        """
+        WARNING: This function is in-development which means it is subject to change. Use at your own risk.
+
+        By default, a source should raise an exception and stop the sync when it encounters an error while syncing a stream. This
+        method can be overridden on a per-source basis so that a source will continue syncing streams other streams even if an
+        exception is raised for a stream.
+        """
+        return False
+
+    @staticmethod
+    def _generate_failed_streams_error_message(stream_failures: Mapping[str, AirbyteTracedException]) -> str:
+        failures = ", ".join([f"{stream}: {exception.__repr__()}" for stream, exception in stream_failures.items()])
+        return f"During the sync, the following streams did not sync successfully: {failures}"

{airbyte-cdk-0.55.5 → airbyte-cdk-0.56.1}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.55.5
+Version: 0.56.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.55.5 → airbyte-cdk-0.56.1}/setup.py

@@ -36,7 +36,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.55.5",
+    version="0.56.1",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

{airbyte-cdk-0.55.5 → airbyte-cdk-0.56.1}/unit_tests/sources/test_abstract_source.py

@@ -111,6 +111,14 @@ class MockStreamOverridesStateMethod(Stream, IncrementalMixin):
         self._cursor_value = value.get(self.cursor_field, self.start_date)
 
 
+class StreamRaisesException(Stream):
+    name = "lamentations"
+    primary_key = None
+
+    def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]:
+        raise AirbyteTracedException(message="I was born only to crash like Icarus")
+
+
 MESSAGE_FROM_REPOSITORY = Mock()
 
 
@@ -1158,3 +1166,97 @@ def test_checkpoint_state_from_stream_instance():
     assert actual_message == _as_state(
         {"teams": {"updated_at": "2022-09-11"}, "managers": {"updated": "expected_here"}}, "managers", {"updated": "expected_here"}
     )
+
+
+def test_continue_sync_with_failed_streams(mocker):
+    """
+    Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure enabled continues
+    syncing even when one stream fails with an error.
+    """
+    stream_output = [{"k1": "v1"}, {"k2": "v2"}]
+    s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
+    s2 = StreamRaisesException()
+    s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
+
+    mocker.patch.object(MockStream, "get_json_schema", return_value={})
+    mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
+
+    src = MockSource(streams=[s1, s2, s3])
+    mocker.patch.object(MockSource, "continue_sync_on_stream_failure", return_value=True)
+    catalog = ConfiguredAirbyteCatalog(
+        streams=[
+            _configured_stream(s1, SyncMode.full_refresh),
+            _configured_stream(s2, SyncMode.full_refresh),
+            _configured_stream(s3, SyncMode.full_refresh),
+        ]
+    )
+
+    expected = _fix_emitted_at(
+        [
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
+            *_as_records("s1", stream_output),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
+            _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
+            _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
+            _as_stream_status("s3", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s3", AirbyteStreamStatus.RUNNING),
+            *_as_records("s3", stream_output),
+            _as_stream_status("s3", AirbyteStreamStatus.COMPLETE),
+        ]
+    )
+
+    messages = []
+    with pytest.raises(AirbyteTracedException) as exc:
+        # We can't use list comprehension or list() here because we are still raising a final exception for the
+        # failed streams and that disrupts parsing the generator into the messages emitted before
+        for message in src.read(logger, {}, catalog):
+            messages.append(message)
+
+    messages = _fix_emitted_at(messages)
+    assert expected == messages
+    assert "lamentations" in exc.value.message
+
+
+def test_stop_sync_with_failed_streams(mocker):
+    """
+    Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure disabled stops
+    syncing once a stream fails with an error.
+    """
+    stream_output = [{"k1": "v1"}, {"k2": "v2"}]
+    s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1")
+    s2 = StreamRaisesException()
+    s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3")
+
+    mocker.patch.object(MockStream, "get_json_schema", return_value={})
+    mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={})
+
+    src = MockSource(streams=[s1, s2, s3])
+    catalog = ConfiguredAirbyteCatalog(
+        streams=[
+            _configured_stream(s1, SyncMode.full_refresh),
+            _configured_stream(s2, SyncMode.full_refresh),
+            _configured_stream(s3, SyncMode.full_refresh),
+        ]
+    )
+
+    expected = _fix_emitted_at(
+        [
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
+            *_as_records("s1", stream_output),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
+            _as_stream_status("lamentations", AirbyteStreamStatus.STARTED),
+            _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE),
+        ]
+    )
+
+    messages = []
+    with pytest.raises(AirbyteTracedException):
+        # We can't use list comprehension or list() here because we are still raising a final exception for the
+        # failed streams and that disrupts parsing the generator into the messages emitted before
+        for message in src.read(logger, {}, catalog):
+            messages.append(message)
+
+    messages = _fix_emitted_at(messages)
+    assert expected == messages