airbyte-cdk 7.3.9.post2.dev18667058048__py3-none-any.whl → 7.3.10.dev0__py3-none-any.whl

airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py

@@ -96,13 +96,12 @@ class ConcurrentReadProcessor:
         """
         stream_name = partition.stream_name()
         self._streams_to_running_partitions[stream_name].add(partition)
-        cursor = self._stream_name_to_instance[stream_name].cursor
         if self._slice_logger.should_log_slice_message(self._logger):
             self._message_repository.emit_message(
                 self._slice_logger.create_slice_log_message(partition.to_slice())
             )
         self._thread_pool_manager.submit(
-            self._partition_reader.process_partition, partition, cursor
+            self._partition_reader.process_partition, partition
         )
 
     def on_partition_complete_sentinel(
@@ -116,16 +115,26 @@ class ConcurrentReadProcessor:
         """
         partition = sentinel.partition
 
-        partitions_running = self._streams_to_running_partitions[partition.stream_name()]
-        if partition in partitions_running:
-            partitions_running.remove(partition)
-            # If all partitions were generated and this was the last one, the stream is done
-            if (
-                partition.stream_name() not in self._streams_currently_generating_partitions
-                and len(partitions_running) == 0
-            ):
-                yield from self._on_stream_is_done(partition.stream_name())
-        yield from self._message_repository.consume_queue()
+        try:
+            if sentinel.is_successful:
+                stream = self._stream_name_to_instance[partition.stream_name()]
+                stream.cursor.close_partition(partition)
+        except Exception as exception:
+            self._flag_exception(partition.stream_name(), exception)
+            yield AirbyteTracedException.from_exception(
+                exception, stream_descriptor=StreamDescriptor(name=partition.stream_name())
+            ).as_sanitized_airbyte_message()
+        finally:
+            partitions_running = self._streams_to_running_partitions[partition.stream_name()]
+            if partition in partitions_running:
+                partitions_running.remove(partition)
+                # If all partitions were generated and this was the last one, the stream is done
+                if (
+                    partition.stream_name() not in self._streams_currently_generating_partitions
+                    and len(partitions_running) == 0
+                ):
+                    yield from self._on_stream_is_done(partition.stream_name())
+            yield from self._message_repository.consume_queue()
 
     def on_record(self, record: Record) -> Iterable[AirbyteMessage]:
         """
@@ -154,6 +163,7 @@ class ConcurrentReadProcessor:
                     stream.as_airbyte_stream(), AirbyteStreamStatus.RUNNING
                 )
             self._record_counter[stream.name] += 1
+            stream.cursor.observe(record)
         yield message
         yield from self._message_repository.consume_queue()
 

airbyte_cdk/sources/streams/concurrent/cursor.py

@@ -198,7 +198,7 @@ class ConcurrentCursor(Cursor):
         # not thread safe. When multiple partitions are being closed by the cursor at the same time, it is
         # possible for one partition to update concurrent_state after a second partition has already read
         # the previous state. This can lead to the second partition overwriting the previous one's state.
-        self._lock = threading.Lock()
+        # self._lock = threading.Lock()
 
     @property
     def state(self) -> MutableMapping[str, Any]:
@@ -273,14 +273,14 @@ class ConcurrentCursor(Cursor):
         return self._connector_state_converter.parse_value(self._cursor_field.extract_value(record))
 
     def close_partition(self, partition: Partition) -> None:
-        with self._lock:
-            slice_count_before = len(self._concurrent_state.get("slices", []))
-            self._add_slice_to_state(partition)
-            if slice_count_before < len(
-                self._concurrent_state["slices"]
-            ):  # only emit if at least one slice has been processed
-                self._merge_partitions()
-                self._emit_state_message()
+        #with self._lock:
+        slice_count_before = len(self._concurrent_state.get("slices", []))
+        self._add_slice_to_state(partition)
+        if slice_count_before < len(
+            self._concurrent_state["slices"]
+        ):  # only emit if at least one slice has been processed
+            self._merge_partitions()
+            self._emit_state_message()
         self._has_closed_at_least_one_slice = True
 
     def _add_slice_to_state(self, partition: Partition) -> None:

airbyte_cdk/sources/streams/concurrent/partition_reader.py

@@ -60,7 +60,7 @@ class PartitionReader:
         self._queue = queue
         self._partition_logger = partition_logger
 
-    def process_partition(self, partition: Partition, cursor: Cursor) -> None:
+    def process_partition(self, partition: Partition) -> None:
         """
         Process a partition and put the records in the output queue.
         When all the partitions are added to the queue, a sentinel is added to the queue to indicate that all the partitions have been generated.
@@ -78,8 +78,6 @@ class PartitionReader:
 
             for record in partition.read():
                 self._queue.put(record)
-                cursor.observe(record)
-            cursor.close_partition(partition)
             self._queue.put(PartitionCompleteSentinel(partition, self._IS_SUCCESSFUL))
         except Exception as e:
             self._queue.put(StreamThreadException(e, partition.stream_name()))

{airbyte_cdk-7.3.9.post2.dev18667058048.dist-info → airbyte_cdk-7.3.10.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 7.3.9.post2.dev18667058048
+Version: 7.3.10.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-7.3.9.post2.dev18667058048.dist-info → airbyte_cdk-7.3.10.dev0.dist-info}/RECORD

@@ -97,7 +97,7 @@ airbyte_cdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/__init__.py,sha256=45J83QsFH3Wky3sVapZWg4C58R_i1thm61M06t2c1AQ,1156
 airbyte_cdk/sources/abstract_source.py,sha256=50vxEBRByiNhT4WJkiFvgM-C6PWqKSJgvuNC_aeg2cw,15547
 airbyte_cdk/sources/concurrent_source/__init__.py,sha256=3D_RJsxQfiLboSCDdNei1Iv-msRp3DXsas6E9kl7dXc,386
-airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py,sha256=qUi9zRjSEOsu3k6niRFjVn4l29OOMZVstIUoqNLw4_k,12208
+airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py,sha256=DPNp3Nh1h_sarXybVMuiIjwlJcIbz0_xAq1sc7mhZvM,12723
 airbyte_cdk/sources/concurrent_source/concurrent_source.py,sha256=2sdgOnWtvClZaibnfn4Yardv2Jjv_lfwMubublDl1ZY,8458
 airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py,sha256=f9PIRPWn2tXu0-bxVeYHL2vYdqCzZ_kgpHg5_Ep-cfQ,6103
 airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py,sha256=z1t-rAZBsqVidv2fpUlPHE9JgyXsITuGk4AMu96mXSQ,696
@@ -353,13 +353,13 @@ airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py,sha256=QTry1QCB
 airbyte_cdk/sources/streams/concurrent/adapters.py,sha256=h4ZewhWn2PzPTt0lZZjcUL4rrpW9E_of7prnI3bm-c4,14004
 airbyte_cdk/sources/streams/concurrent/availability_strategy.py,sha256=M0XmvF3vjlr4GbCM0XH1hAj7udiAONM9SnmXjqufzLM,1035
 airbyte_cdk/sources/streams/concurrent/clamping.py,sha256=i26GVyui2ScEXSP-IP_61K2HaTp1-6lTlYHsZVYpuZA,3240
-airbyte_cdk/sources/streams/concurrent/cursor.py,sha256=oEcqUyswPyOg6SnjrFr8c8YYxVvaaInWNCHRLQcKjmk,25713
+airbyte_cdk/sources/streams/concurrent/cursor.py,sha256=WAJAft9GSliVXWOQVjK4Aya5FdunjoFz5j0Km5l2MDY,25688
 airbyte_cdk/sources/streams/concurrent/cursor_types.py,sha256=ZyWLPpeLX1qXcP5MwS-wxK11IBMsnVPCw9zx8gA2_Ro,843
 airbyte_cdk/sources/streams/concurrent/default_stream.py,sha256=SSufbo5f7OOYS8DZaABXeJVvodcfp9wb8J9lT5Xik3s,4744
 airbyte_cdk/sources/streams/concurrent/exceptions.py,sha256=JOZ446MCLpmF26r9KfS6OO_6rGjcjgJNZdcw6jccjEI,468
 airbyte_cdk/sources/streams/concurrent/helpers.py,sha256=S6AW8TgIASCZ2UuUcQLE8OzgYUHWt2-KPOvNPwnQf-Q,1596
 airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py,sha256=2t64b_z9cEPmlHZnjSiMTO8PEtEdiAJDG0JcYOtUqAE,3363
-airbyte_cdk/sources/streams/concurrent/partition_reader.py,sha256=qKd1NBlAjwo4mRVq9bowWr7oYg8SpgISklpkjUkie4k,3434
+airbyte_cdk/sources/streams/concurrent/partition_reader.py,sha256=b8gTenOot1Yg5aer70onNLyCk43nlBZtBvBRKjW8VHs,3333
 airbyte_cdk/sources/streams/concurrent/partitions/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/streams/concurrent/partitions/partition.py,sha256=CmaRcKn8y118No3qvbRV9DBeAUKv17lrVgloR4Y9TwU,1490
 airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py,sha256=_ymkkBr71_qt1fW0_MUqw96OfNBkeJngXQ09yolEDHw,441
@@ -459,9 +459,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=9YDJmnIGFsT51CVQf2tSSvTapGimITjEFGbUTSZAGTI,963
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-7.3.9.post2.dev18667058048.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-7.3.9.post2.dev18667058048.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-7.3.9.post2.dev18667058048.dist-info/METADATA,sha256=qXNgvIOFA63FeGgWN7HJ-e0SfQA9if25LIFffhynZOM,6780
-airbyte_cdk-7.3.9.post2.dev18667058048.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-7.3.9.post2.dev18667058048.dist-info/entry_points.txt,sha256=eLZ2UYvJZGm1s07Pplcs--1Gim60YhZWTb53j_dghwU,195
-airbyte_cdk-7.3.9.post2.dev18667058048.dist-info/RECORD,,
+airbyte_cdk-7.3.10.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-7.3.10.dev0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-7.3.10.dev0.dist-info/METADATA,sha256=NOmBy-jxE-RCgk5KiN_Tg-3pDHgrKUaBQwnxlNRevos,6765
+airbyte_cdk-7.3.10.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-7.3.10.dev0.dist-info/entry_points.txt,sha256=eLZ2UYvJZGm1s07Pplcs--1Gim60YhZWTb53j_dghwU,195
+airbyte_cdk-7.3.10.dev0.dist-info/RECORD,,