airbyte-cdk 0.53.3__py3-none-any.whl → 0.53.4__py3-none-any.whl

airbyte_cdk/sources/streams/concurrent/thread_based_concurrent_stream.py

@@ -154,7 +154,7 @@ class ThreadBasedConcurrentStream(AbstractStream):
         if len(futures) < self._max_concurrent_tasks:
             return
 
-        for index in range(len(futures)):
+        for index in reversed(range(len(futures))):
             future = futures[index]
             optional_exception = future.exception()
             if optional_exception:

{airbyte_cdk-0.53.3.dist-info → airbyte_cdk-0.53.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.53.3
+Version: 0.53.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.53.3.dist-info → airbyte_cdk-0.53.4.dist-info}/RECORD

@@ -199,7 +199,7 @@ airbyte_cdk/sources/streams/concurrent/exceptions.py,sha256=-WETGIY5_QFmVeDFiqm4
 airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py,sha256=uAUhCkxFOaptDJfIEDmFnnF4xn4coG5kvE4B_5tVx14,1557
 airbyte_cdk/sources/streams/concurrent/partition_reader.py,sha256=H8sGVVGx6uKMSUehRaqmVbE19DE3cx3NivQ4sFj8wbk,1303
 airbyte_cdk/sources/streams/concurrent/state_converter.py,sha256=PwqcRVPR6LQxWL0yvPTp_u2Uh0hBJU-BDSjPKiyJVEk,4689
-airbyte_cdk/sources/streams/concurrent/thread_based_concurrent_stream.py,sha256=ZfjrqY-I43H-qQHmwJnHnP_4snvPBFwD9dIVwV1gOqU,10833
+airbyte_cdk/sources/streams/concurrent/thread_based_concurrent_stream.py,sha256=M7CpPPBswHTYjG4opiTOf5eWHOJ6i4TyP0v991pFxOo,10843
 airbyte_cdk/sources/streams/concurrent/partitions/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/streams/concurrent/partitions/partition.py,sha256=tjXF8lZMvyfZaCYCHr5aTPwbVstmRjYZDwYAvLDY-ds,1312
 airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py,sha256=_ymkkBr71_qt1fW0_MUqw96OfNBkeJngXQ09yolEDHw,441
@@ -387,7 +387,7 @@ unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py,sha
 unit_tests/sources/streams/concurrent/test_cursor.py,sha256=sqH8xE3GamETSSVqsdKafziAAm-y_j_MegES_C5ExMM,5790
 unit_tests/sources/streams/concurrent/test_partition_reader.py,sha256=eM5dzfmLKm9Lj-BfQUjAZRhCZzfvhk7AkKpcHGcoPfg,931
 unit_tests/sources/streams/concurrent/test_state_converter.py,sha256=rvg8becWR1iPdm5TAanZssKj5_iw8dInE_uqmjqghZE,8349
-unit_tests/sources/streams/concurrent/test_thread_based_concurrent_stream.py,sha256=HJYZCwgSyGmKdQSInZK1rUDbtW5RepdOa05hC10RPe4,10894
+unit_tests/sources/streams/concurrent/test_thread_based_concurrent_stream.py,sha256=_jBMJIZ6Hu9mWX4v9SRUdtxvgntA-rQpNbbygBi6HXA,11629
 unit_tests/sources/streams/concurrent/scenarios/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py,sha256=x77AQf8_O4dQ2aF1o800CzI0hOEyU8ayxoNdSOvxkhM,10495
 unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=FdgEO-bWA_IDFIJb0W83qE4QCCZ8eexbn_Mq8LJq0iE,5040
@@ -411,8 +411,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
 unit_tests/utils/test_stream_status_utils.py,sha256=N2TxwKge45RHUKFlPcP2o5jXYjJPKMKiu6Fm2_leZYY,3388
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.53.3.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.53.3.dist-info/METADATA,sha256=9cv_fGW7SUbezVGIwe5oJYruTMWoPRAaD6rWassO2IU,11983
-airbyte_cdk-0.53.3.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-airbyte_cdk-0.53.3.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.53.3.dist-info/RECORD,,
+airbyte_cdk-0.53.4.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.53.4.dist-info/METADATA,sha256=n_0ZAbpdj_k3YO8y-B1g9javdvN876EaSuA6-Cu5xQY,11983
+airbyte_cdk-0.53.4.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
+airbyte_cdk-0.53.4.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.53.4.dist-info/RECORD,,

unit_tests/sources/streams/concurrent/test_thread_based_concurrent_stream.py

@@ -13,6 +13,8 @@ from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partitio
 from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
 from airbyte_cdk.sources.streams.concurrent.thread_based_concurrent_stream import ThreadBasedConcurrentStream
 
+_MAX_CONCURRENT_TASKS = 2
+
 
 class ThreadBasedConcurrentStreamTest(unittest.TestCase):
     def setUp(self):
@@ -39,7 +41,7 @@ class ThreadBasedConcurrentStreamTest(unittest.TestCase):
             self._logger,
             self._message_repository,
             1,
-            2,
+            _MAX_CONCURRENT_TASKS,
             0,
             cursor=self._cursor,
         )
@@ -142,15 +144,33 @@ class ThreadBasedConcurrentStreamTest(unittest.TestCase):
         f2 = Mock()
 
         # Verify that the done() method will be called until only one future is still running
-        f1.done.return_value = False
+        f1.done.return_value = True
         f1.exception.return_value = None
-        f2.done.return_value = False
-        f2.exception.return_value = ValueError("An exception")
+        f2.done.return_value = True
+        f2.exception.return_value = ValueError("ERROR")
         futures = [f1, f2]
 
         with pytest.raises(RuntimeError):
             self._stream._wait_while_too_many_pending_futures(futures)
 
+    def test_given_removing_multiple_elements_when_pruning_then_fail_immediately(self):
+        # Verify that the done() method will be called until only one future is still running
+        futures = []
+        for _ in range(_MAX_CONCURRENT_TASKS + 1):
+            future = Mock()
+            future.done.return_value = True
+            future.exception.return_value = None
+            futures.append(future)
+
+        pending_future = Mock()
+        pending_future.done.return_value = False
+        pending_future.exception.return_value = None
+        futures.append(pending_future)
+
+        self._stream._wait_while_too_many_pending_futures(futures)
+
+        assert futures == [pending_future]
+
     def test_as_airbyte_stream(self):
         expected_airbyte_stream = AirbyteStream(
             name=self._name,