airbyte-cdk 0.58.8__py3-none-any.whl → 0.59.0__py3-none-any.whl

{airbyte_cdk-0.58.8.dist-info → airbyte_cdk-0.59.0.dist-info}/RECORD

@@ -31,11 +31,12 @@ airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZ
 airbyte_cdk/sources/http_logger.py,sha256=v0kkpDtA0GUOgj6_3AayrYaBrSHBqG4t3MGbrtxaNmU,1437
 airbyte_cdk/sources/source.py,sha256=dk50z8Roc28MJ8FxWe652B-GwItO__bTZqFm7WOtHnw,4412
 airbyte_cdk/sources/concurrent_source/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py,sha256=7A8bdOqg9Hpw37QgqoGO6eYBx8_pKikt_AJUcImp-x4,9715
-airbyte_cdk/sources/concurrent_source/concurrent_source.py,sha256=qdxeuZJh5cW1THyASfCBJ_H92C42bQz0vv17IhXp7l0,7836
+airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py,sha256=TIqzf-_fYfCe4n77UwREiXLnFfSlTT-zsHct760KZw8,10052
+airbyte_cdk/sources/concurrent_source/concurrent_source.py,sha256=Rg8_62UfOIQd1IE8ILpj_z4xNAD9vtIrNRVvMgHMfdA,7957
 airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py,sha256=si5ipxvzCE2Pdusg19evr8ziG7eqBBuBuDPww6i3Amg,3223
 airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py,sha256=oExaUlnDepGZjNmauIkFDCbWtxZvkBCFo1K0wAr4sRA,493
-airbyte_cdk/sources/concurrent_source/thread_pool_manager.py,sha256=huFp0uuG2kZAnbHY8oeDYuX0hfmP-rLOnJMa72ZuWt0,3905
+airbyte_cdk/sources/concurrent_source/thread_pool_manager.py,sha256=M-IcXJwMBK31Luk-SLQSFwTV8PelDdaSFMLlB6K1T4w,3908
+airbyte_cdk/sources/concurrent_source/throttler.py,sha256=gAdwWTk5iveSRUceXIYAms8v1qGSJytzUD07lkR1I9k,926
 airbyte_cdk/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/create_partial.py,sha256=sUJOwD8hBzW4pxw2XhYlSTMgl-WMc5WpP5Oq_jo3fHw,3371
 airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=4KtVA54eDmZt8JAFuHLwQv8PmhOmiiaGRBNUgcAQBzQ,87107
@@ -201,19 +202,20 @@ airbyte_cdk/sources/streams/concurrent/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuY
 airbyte_cdk/sources/streams/concurrent/abstract_stream.py,sha256=W7WEz6FrfAjb0o_msnMBIESSVO1qJC2_A8ocYg55Rw4,3579
 airbyte_cdk/sources/streams/concurrent/adapters.py,sha256=f48kLzOHYNeD7Tfsdy7WaZ__hB24SfCTcW5WpQedqTc,18648
 airbyte_cdk/sources/streams/concurrent/availability_strategy.py,sha256=8xDRpfktnARBbRi_RwznvKuoGrpPF2b6tQyloMwogkM,2013
-airbyte_cdk/sources/streams/concurrent/cursor.py,sha256=GnSRDkEEvg2GNy_fEc9cWFoYI1oEfvzwg1vhRrusqWg,7105
+airbyte_cdk/sources/streams/concurrent/cursor.py,sha256=_mAbnJILeiGOGBNXeeXrSSoz7rveEBMoL97569EPEBY,8106
 airbyte_cdk/sources/streams/concurrent/default_stream.py,sha256=w83pvFbw9vjfhbovw-LrCFiwQMO8hfo1Vm-1CB1SeXQ,2777
 airbyte_cdk/sources/streams/concurrent/exceptions.py,sha256=-WETGIY5_QFmVeDFiqm4WhRJ_nNCkfcDwOQqx6cSqrI,365
-airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py,sha256=Mmn0hYq2xWe2a0WOpZPF3iZNozfmv7vY37LgfdY7DVo,1570
-airbyte_cdk/sources/streams/concurrent/partition_reader.py,sha256=H8sGVVGx6uKMSUehRaqmVbE19DE3cx3NivQ4sFj8wbk,1303
+airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py,sha256=TicEVRyLt5Y85xa8bXsrGzjmNMJAulBLqe1LfBLnHmk,1540
+airbyte_cdk/sources/streams/concurrent/partition_reader.py,sha256=KHFYK4Yn_6L1gmt31N62K1z28evPKtgI_tkhKQ5oWCg,1358
 airbyte_cdk/sources/streams/concurrent/partitions/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/streams/concurrent/partitions/partition.py,sha256=o2QvDYZF3Tn9NbC5jc1UkDwMiCWq9fNGj493u2WFoko,1795
 airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py,sha256=_ymkkBr71_qt1fW0_MUqw96OfNBkeJngXQ09yolEDHw,441
 airbyte_cdk/sources/streams/concurrent/partitions/record.py,sha256=-Q3zLex3CHOXiB-KOZLbBZaPiQ_BLFJdknr6yoRz9I0,600
+airbyte_cdk/sources/streams/concurrent/partitions/throttled_queue.py,sha256=P6KrMb4GtcDUbMcx7pVb7dfF_igeW9Utn2MFoVHkH6o,1589
 airbyte_cdk/sources/streams/concurrent/partitions/types.py,sha256=iVARnsGOSdvlSCqAf-yxc4_PUT3oOR9B6cyVNcLTjY8,932
 airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=8_97TvqOSn5jxmHsNMmnFi_A4XxGtMrX5GG6eNOyLtA,2954
-airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py,sha256=4K_Fa0nviG3L3Mu4YX3oEs1cAfE0QqzrKkkdJsL21M4,7002
+airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=jeVP3V9uDM_aCMh1G3kZNKafjopy1rZzIAJ8sU-69KU,2613
+airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py,sha256=BvVwPiVHE-R7mWDjcqhsKnZyD6OcpmAYaih7pcmOUvo,7589
 airbyte_cdk/sources/streams/http/__init__.py,sha256=cTP2d7Wf0hYXaN20U0dtxKa1pFZ9rI-lBbkQ0UM1apQ,261
 airbyte_cdk/sources/streams/http/availability_strategy.py,sha256=MHgW42gwaevaCVnNLrUSE6WJHT4reeZ417nMWrmbC7o,6884
 airbyte_cdk/sources/streams/http/exceptions.py,sha256=OokLDI7W8hZvq9e15sL3em2AdwmzmcAl72Ms-i5l0Nw,1334
@@ -407,16 +409,18 @@ unit_tests/sources/streams/test_stream_read.py,sha256=xxyYV5jPsAptmI0awPO_VGWMaE
 unit_tests/sources/streams/test_streams_core.py,sha256=YOC7XqWFJ13Z4YuO9Nh4AR4AwpJ-s111vqPplFfpxk4,5059
 unit_tests/sources/streams/concurrent/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 unit_tests/sources/streams/concurrent/test_adapters.py,sha256=Y_c1vKCtGKEzrUSncmpgp0lgFnArmBrIrmLFaOIAxRg,15439
-unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py,sha256=v8yf19_sDgVcWop6WKotahlQiO6B8MwxhGi3AL4vHm8,1375
-unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py,sha256=1578s0CtFVQALrv2slo12RIgNmNwwobJAFWfBre8jdc,23822
-unit_tests/sources/streams/concurrent/test_cursor.py,sha256=ere6x4nhhtSmEdvDoksLvLJTFkcl29N7VI7Mj6jin-I,5385
-unit_tests/sources/streams/concurrent/test_datetime_state_converter.py,sha256=mzlm8JL5AuYmNX4r9VViLKNaU1zUyBvBQ3lIdChW9pU,19153
+unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py,sha256=a5JWWWc20zeEGmMzYzzk-_6XwfDEZOGgW287dVgft_8,1339
+unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py,sha256=Dc7PGfQge1ymxnaTlPKGMVOLCV81JCXoK1ciJPwHIhg,26347
+unit_tests/sources/streams/concurrent/test_cursor.py,sha256=0nFp9xauWxDVxJmFQLedvnxWKDoAlk9ChDkd0Mmrhm4,5951
+unit_tests/sources/streams/concurrent/test_datetime_state_converter.py,sha256=vx-oPmGSzSfBM37ZXN_wXLeTOlozErqQoa5sc5zP42o,13287
 unit_tests/sources/streams/concurrent/test_default_stream.py,sha256=VLF46ESoRqcoALYCdrdZ2NDl5s2T1fRRWsYAy2-IwYw,6502
 unit_tests/sources/streams/concurrent/test_partition_reader.py,sha256=2uj7uV3ie0BMb--aa3MUru-f4jLiYUR-Nl0r3EhwxLQ,951
-unit_tests/sources/streams/concurrent/test_thread_pool_manager.py,sha256=7L9Sv7VXULOHx3-KSFwFtzAY1X96wcPiPKGq38BQEVg,3699
+unit_tests/sources/streams/concurrent/test_thread_pool_manager.py,sha256=UzlMhXTgXAuqqPrESGjkDG9JLj4UdPo2bx3T9oLCFpA,3140
+unit_tests/sources/streams/concurrent/test_throttled_queue.py,sha256=05NgNkx5c5rL8RE2ViOdJz8jty5-67BQPvJNLpVjuMo,1747
+unit_tests/sources/streams/concurrent/test_throttler.py,sha256=y1cWUdKZP5iy4FKWmGdgEk4e_0WY0VNgRvzmlU114NY,448
 unit_tests/sources/streams/concurrent/scenarios/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py,sha256=TH4vzdHNWvw4JsF0v4n6wrR1Rnr-WfU3R6nnOwGLNwg,9751
-unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=cFqcfrk-P4Gz-5yUFjTXIZ5U0zhzg3jayZiF6W79oPc,5880
+unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=BGqaYgU_ow4PsuhDjFwAFkU1VCkUuromvTAUV5tOaJ8,5816
 unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py,sha256=kDKKV0ApASyS5c2HYkKvYohSkT--46TqALirqU8POjg,13804
 unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py,sha256=Z_4-ClsxBupmN7Pbl8lF9bkSA9wnjLtrgA9WR_8VRi8,3757
 unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py,sha256=KqCLsXB_9rV4hNdSPrNynK3G-UIsipqsZT6X0Z-iM5E,13175
@@ -444,8 +448,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
 unit_tests/utils/test_stream_status_utils.py,sha256=Xr8MZ2HWgTVIyMbywDvuYkRaUF4RZLQOT8-JjvcfR24,2970
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.58.8.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.58.8.dist-info/METADATA,sha256=dc4NGd9LcXAFFyYyc-NiTyDUkWjJsvz9iztzMveJZUA,11073
-airbyte_cdk-0.58.8.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-airbyte_cdk-0.58.8.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.58.8.dist-info/RECORD,,
+airbyte_cdk-0.59.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.59.0.dist-info/METADATA,sha256=5wiAdalwK0NUdE--8UKtiGbRy9ccsaY1T255M_Ei850,11073
+airbyte_cdk-0.59.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+airbyte_cdk-0.59.0.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.59.0.dist-info/RECORD,,

unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py

@@ -52,8 +52,7 @@ class StreamFacadeSource(ConcurrentSourceAdapter):
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         state_manager = ConnectorStateManager(stream_instance_map={s.name: s for s in self._streams}, state=self._state)
         state_converter = StreamFacadeConcurrentConnectorStateConverter()
-        stream_states = [state_converter.get_concurrent_stream_state(self._cursor_field, state_manager.get_stream_state(stream.name, stream.namespace))
-                         for stream in self._streams]
+        stream_states = [state_manager.get_stream_state(stream.name, stream.namespace) for stream in self._streams]
         return [
             StreamFacade.create_from_stream(
                 stream,
@@ -69,6 +68,7 @@ class StreamFacadeSource(ConcurrentSourceAdapter):
                     state_converter,
                     self._cursor_field,
                     self._cursor_boundaries,
+                    None,
                 )
                 if self._cursor_field
                 else NoopCursor(),

unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py

@@ -2,21 +2,21 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from queue import Queue
-from unittest.mock import Mock
+from unittest.mock import Mock, call
 
 import pytest
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel
 from airbyte_cdk.sources.streams.concurrent.adapters import StreamPartition
 from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer
+from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue
 
 
 @pytest.mark.parametrize(
     "slices", [pytest.param([], id="test_no_partitions"), pytest.param([{"partition": 1}, {"partition": 2}], id="test_two_partitions")]
 )
 def test_partition_generator(slices):
-    queue = Queue()
+    queue = Mock(spec=ThrottledQueue)
     partition_generator = PartitionEnqueuer(queue)
 
     stream = Mock()
@@ -30,10 +30,4 @@ def test_partition_generator(slices):
 
     partition_generator.generate_partitions(stream)
 
-    actual_partitions = []
-    while partition := queue.get(False):
-        if isinstance(partition, PartitionGenerationCompletedSentinel):
-            break
-        actual_partitions.append(partition)
-
-    assert actual_partitions == partitions
+    assert queue.put.has_calls([call(p) for p in partitions] + [call(PartitionGenerationCompletedSentinel(stream))])

unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py

@@ -40,13 +40,11 @@ class TestConcurrentReadProcessor(unittest.TestCase):
         self._thread_pool_manager = Mock(spec=ThreadPoolManager)
 
         self._an_open_partition = Mock(spec=Partition)
-        self._an_open_partition.is_closed.return_value = False
         self._log_message = Mock(spec=LogMessage)
         self._an_open_partition.to_slice.return_value = self._log_message
         self._an_open_partition.stream_name.return_value = _STREAM_NAME
 
         self._a_closed_partition = Mock(spec=Partition)
-        self._a_closed_partition.is_closed.return_value = True
         self._a_closed_partition.stream_name.return_value = _ANOTHER_STREAM_NAME
 
         self._logger = Mock(spec=logging.Logger)
@@ -76,6 +74,19 @@ class TestConcurrentReadProcessor(unittest.TestCase):
         self._record.stream_name = _STREAM_NAME
         self._record.data = self._record_data
 
+    def test_stream_is_not_done_initially(self):
+        stream_instances_to_read_from = [self._stream]
+        handler = ConcurrentReadProcessor(
+            stream_instances_to_read_from,
+            self._partition_enqueuer,
+            self._thread_pool_manager,
+            self._logger,
+            self._slice_logger,
+            self._message_repository,
+            self._partition_reader,
+        )
+        assert not handler._is_stream_done(self._stream.name)
+
     def test_handle_partition_done_no_other_streams_to_generate_partitions_for(self):
         stream_instances_to_read_from = [self._stream]
 
@@ -111,7 +122,6 @@ class TestConcurrentReadProcessor(unittest.TestCase):
             self._partition_reader,
         )
         handler.start_next_partition_generator()
-        handler.on_partition(self._a_closed_partition)
 
         sentinel = PartitionGenerationCompletedSentinel(self._another_stream)
         messages = handler.on_partition_generation_completed(sentinel)
@@ -147,7 +157,7 @@ class TestConcurrentReadProcessor(unittest.TestCase):
         handler.on_partition(self._a_closed_partition)
 
         self._thread_pool_manager.submit.assert_called_with(self._partition_reader.process_partition, self._a_closed_partition)
-        assert self._a_closed_partition in handler._streams_to_partitions[_ANOTHER_STREAM_NAME]
+        assert self._a_closed_partition in handler._streams_to_running_partitions[_ANOTHER_STREAM_NAME]
 
     def test_handle_partition_emits_log_message_if_it_should_be_logged(self):
         stream_instances_to_read_from = [self._stream]
@@ -169,15 +179,16 @@ class TestConcurrentReadProcessor(unittest.TestCase):
 
         self._thread_pool_manager.submit.assert_called_with(self._partition_reader.process_partition, self._an_open_partition)
         self._message_repository.emit_message.assert_called_with(self._log_message)
-        assert self._an_open_partition in handler._streams_to_partitions[_STREAM_NAME]
 
+        assert self._an_open_partition in handler._streams_to_running_partitions[_STREAM_NAME]
+
+    @freezegun.freeze_time("2020-01-01T00:00:00")
     def test_handle_on_partition_complete_sentinel_with_messages_from_repository(self):
         stream_instances_to_read_from = [self._stream]
         partition = Mock(spec=Partition)
         log_message = Mock(spec=LogMessage)
         partition.to_slice.return_value = log_message
         partition.stream_name.return_value = _STREAM_NAME
-        partition.is_closed.return_value = True
 
         handler = ConcurrentReadProcessor(
             stream_instances_to_read_from,
@@ -189,6 +200,7 @@ class TestConcurrentReadProcessor(unittest.TestCase):
             self._partition_reader,
         )
         handler.start_next_partition_generator()
+        handler.on_partition(partition)
 
         sentinel = PartitionCompleteSentinel(partition)
 
@@ -223,6 +235,7 @@ class TestConcurrentReadProcessor(unittest.TestCase):
             self._partition_reader,
         )
         handler.start_next_partition_generator()
+        handler.on_partition(self._a_closed_partition)
         handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._another_stream))
 
         sentinel = PartitionCompleteSentinel(self._a_closed_partition)
@@ -254,7 +267,6 @@ class TestConcurrentReadProcessor(unittest.TestCase):
         log_message = Mock(spec=LogMessage)
         partition.to_slice.return_value = log_message
         partition.stream_name.return_value = _STREAM_NAME
-        partition.is_closed.return_value = True
 
         handler = ConcurrentReadProcessor(
             stream_instances_to_read_from,
@@ -282,7 +294,6 @@ class TestConcurrentReadProcessor(unittest.TestCase):
         log_message = Mock(spec=LogMessage)
         partition.to_slice.return_value = log_message
         partition.stream_name.return_value = _STREAM_NAME
-        partition.is_closed.return_value = True
         self._message_repository.consume_queue.return_value = []
 
         handler = ConcurrentReadProcessor(
@@ -319,7 +330,6 @@ class TestConcurrentReadProcessor(unittest.TestCase):
         log_message = Mock(spec=LogMessage)
         partition.to_slice.return_value = log_message
         partition.stream_name.return_value = _STREAM_NAME
-        partition.is_closed.return_value = True
         slice_logger = Mock(spec=SliceLogger)
         slice_logger.should_log_slice_message.return_value = True
         slice_logger.create_slice_log_message.return_value = log_message
@@ -370,7 +380,6 @@ class TestConcurrentReadProcessor(unittest.TestCase):
         stream_instances_to_read_from = [self._stream]
         partition = Mock(spec=Partition)
         partition.stream_name.return_value = _STREAM_NAME
-        partition.is_closed.return_value = True
 
         handler = ConcurrentReadProcessor(
             stream_instances_to_read_from,
@@ -413,7 +422,6 @@ class TestConcurrentReadProcessor(unittest.TestCase):
         log_message = Mock(spec=LogMessage)
         partition.to_slice.return_value = log_message
         partition.stream_name.return_value = _STREAM_NAME
-        partition.is_closed.return_value = True
         self._message_repository.consume_queue.return_value = [
             AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=LogLevel.INFO, message="message emitted from the repository"))
         ]
@@ -474,7 +482,69 @@ class TestConcurrentReadProcessor(unittest.TestCase):
             self._message_repository,
             self._partition_reader,
         )
-        handler._streams_to_partitions = {_STREAM_NAME: {self._an_open_partition}, _ANOTHER_STREAM_NAME: {self._a_closed_partition}}
+
+        handler.start_next_partition_generator()
+
+        another_stream = Mock(spec=AbstractStream)
+        another_stream.name = _STREAM_NAME
+        another_stream.as_airbyte_stream.return_value = AirbyteStream(
+            name=_ANOTHER_STREAM_NAME,
+            json_schema={},
+            supported_sync_modes=[SyncMode.full_refresh],
+        )
+
+        exception = RuntimeError("Something went wrong")
+
+        messages = []
+
+        with self.assertRaises(RuntimeError):
+            for m in handler.on_exception(exception):
+                messages.append(m)
+
+        expected_message = [
+            AirbyteMessage(
+                type=MessageType.TRACE,
+                trace=AirbyteTraceMessage(
+                    type=TraceType.STREAM_STATUS,
+                    emitted_at=1577836800000.0,
+                    stream_status=AirbyteStreamStatusTraceMessage(
+                        stream_descriptor=StreamDescriptor(name=_STREAM_NAME), status=AirbyteStreamStatus(AirbyteStreamStatus.INCOMPLETE)
+                    ),
+                ),
+            ),
+            AirbyteMessage(
+                type=MessageType.TRACE,
+                trace=AirbyteTraceMessage(
+                    type=TraceType.STREAM_STATUS,
+                    emitted_at=1577836800000.0,
+                    stream_status=AirbyteStreamStatusTraceMessage(
+                        stream_descriptor=StreamDescriptor(name=_ANOTHER_STREAM_NAME), status=AirbyteStreamStatus(AirbyteStreamStatus.INCOMPLETE)
+                    ),
+                ),
+            )
+        ]
+
+        assert messages == expected_message
+        self._thread_pool_manager.shutdown.assert_called_once()
+
+    @freezegun.freeze_time("2020-01-01T00:00:00")
+    def test_on_exception_does_not_stop_streams_that_are_already_done(self):
+        stream_instances_to_read_from = [self._stream, self._another_stream]
+
+        handler = ConcurrentReadProcessor(
+            stream_instances_to_read_from,
+            self._partition_enqueuer,
+            self._thread_pool_manager,
+            self._logger,
+            self._slice_logger,
+            self._message_repository,
+            self._partition_reader,
+        )
+
+        handler.start_next_partition_generator()
+        handler.on_partition(self._an_open_partition)
+        handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._stream))
+        handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._another_stream))
 
         another_stream = Mock(spec=AbstractStream)
         another_stream.name = _STREAM_NAME

unit_tests/sources/streams/concurrent/test_cursor.py

@@ -45,33 +45,50 @@ class ConcurrentCursorTest(TestCase):
         return ConcurrentCursor(
             _A_STREAM_NAME,
             _A_STREAM_NAMESPACE,
-            self._state_converter.get_concurrent_stream_state(CursorField(_A_CURSOR_FIELD_KEY), {}),
+            {},
             self._message_repository,
             self._state_manager,
             self._state_converter,
             CursorField(_A_CURSOR_FIELD_KEY),
             _SLICE_BOUNDARY_FIELDS,
+            None,
         )
 
     def _cursor_without_slice_boundary_fields(self) -> ConcurrentCursor:
         return ConcurrentCursor(
             _A_STREAM_NAME,
             _A_STREAM_NAMESPACE,
-            self._state_converter.get_concurrent_stream_state(CursorField(_A_CURSOR_FIELD_KEY), {}),
+            {},
             self._message_repository,
             self._state_manager,
             self._state_converter,
             CursorField(_A_CURSOR_FIELD_KEY),
             None,
+            None,
         )
 
     def test_given_boundary_fields_when_close_partition_then_emit_state(self) -> None:
-        self._cursor_with_slice_boundary_fields().close_partition(
+        cursor = self._cursor_with_slice_boundary_fields()
+        cursor.close_partition(
             _partition(
                 {_LOWER_SLICE_BOUNDARY_FIELD: 12, _UPPER_SLICE_BOUNDARY_FIELD: 30},
             )
         )
 
+        self._message_repository.emit_message.assert_called_once_with(self._state_manager.create_state_message.return_value)
+        self._state_manager.update_state_for_stream.assert_called_once_with(
+            _A_STREAM_NAME,
+            _A_STREAM_NAMESPACE,
+            {_A_CURSOR_FIELD_KEY: 0},  # State message is updated to the legacy format before being emitted
+        )
+
+    def test_given_boundary_fields_when_close_partition_then_emit_updated_state(self) -> None:
+        self._cursor_with_slice_boundary_fields().close_partition(
+            _partition(
+                {_LOWER_SLICE_BOUNDARY_FIELD: 0, _UPPER_SLICE_BOUNDARY_FIELD: 30},
+            )
+        )
+
         self._message_repository.emit_message.assert_called_once_with(self._state_manager.create_state_message.return_value)
         self._state_manager.update_state_for_stream.assert_called_once_with(
             _A_STREAM_NAME,