airbyte-cdk 0.58.8__tar.gz → 0.59.0__tar.gz

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.58.8
+Version: 0.59.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py

@@ -43,9 +43,9 @@ class ConcurrentReadProcessor:
         """
         self._stream_name_to_instance = {s.name: s for s in stream_instances_to_read_from}
         self._record_counter = {}
-        self._streams_to_partitions: Dict[str, Set[Partition]] = {}
+        self._streams_to_running_partitions: Dict[str, Set[Partition]] = {}
         for stream in stream_instances_to_read_from:
-            self._streams_to_partitions[stream.name] = set()
+            self._streams_to_running_partitions[stream.name] = set()
             self._record_counter[stream.name] = 0
         self._thread_pool_manager = thread_pool_manager
         self._partition_enqueuer = partition_enqueuer
@@ -55,6 +55,7 @@ class ConcurrentReadProcessor:
         self._slice_logger = slice_logger
         self._message_repository = message_repository
         self._partition_reader = partition_reader
+        self._streams_done: Set[str] = set()
 
     def on_partition_generation_completed(self, sentinel: PartitionGenerationCompletedSentinel) -> Iterable[AirbyteMessage]:
         """
@@ -67,7 +68,8 @@ class ConcurrentReadProcessor:
         self._streams_currently_generating_partitions.remove(sentinel.stream.name)
         ret = []
         # It is possible for the stream to already be done if no partitions were generated
-        if self._is_stream_done(stream_name):
+        # If the partition generation process was completed and there are no partitions left to process, the stream is done
+        if self._is_stream_done(stream_name) or len(self._streams_to_running_partitions[stream_name]) == 0:
             ret.append(self._on_stream_is_done(stream_name))
         if self._stream_instances_to_start_partition_generation:
             ret.append(self.start_next_partition_generator())
@@ -81,7 +83,7 @@ class ConcurrentReadProcessor:
         3. Submit the partition to the thread pool manager
         """
         stream_name = partition.stream_name()
-        self._streams_to_partitions[stream_name].add(partition)
+        self._streams_to_running_partitions[stream_name].add(partition)
         if self._slice_logger.should_log_slice_message(self._logger):
             self._message_repository.emit_message(self._slice_logger.create_slice_log_message(partition.to_slice()))
         self._thread_pool_manager.submit(self._partition_reader.process_partition, partition)
@@ -95,8 +97,12 @@ class ConcurrentReadProcessor:
         """
         partition = sentinel.partition
         partition.close()
-        if self._is_stream_done(partition.stream_name()):
-            yield self._on_stream_is_done(partition.stream_name())
+        partitions_running = self._streams_to_running_partitions[partition.stream_name()]
+        if partition in partitions_running:
+            partitions_running.remove(partition)
+            # If all partitions were generated and this was the last one, the stream is done
+            if partition.stream_name() not in self._streams_currently_generating_partitions and len(partitions_running) == 0:
+                yield self._on_stream_is_done(partition.stream_name())
         yield from self._message_repository.consume_queue()
 
     def on_record(self, record: Record) -> Iterable[AirbyteMessage]:
@@ -114,11 +120,10 @@ class ConcurrentReadProcessor:
         message = stream_data_to_airbyte_message(record.stream_name, record.data)
         stream = self._stream_name_to_instance[record.stream_name]
 
-        if self._record_counter[stream.name] == 0:
-            self._logger.info(f"Marking stream {stream.name} as RUNNING")
-            yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.RUNNING)
-
         if message.type == MessageType.RECORD:
+            if self._record_counter[stream.name] == 0:
+                self._logger.info(f"Marking stream {stream.name} as RUNNING")
+                yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.RUNNING)
             self._record_counter[stream.name] += 1
         yield message
         yield from self._message_repository.consume_queue()
@@ -161,30 +166,24 @@ class ConcurrentReadProcessor:
         2. There are no more streams to read from
         3. All partitions for all streams are closed
         """
-        return (
-            not self._streams_currently_generating_partitions
-            and not self._stream_instances_to_start_partition_generation
-            and all([all(p.is_closed() for p in partitions) for partitions in self._streams_to_partitions.values()])
-        )
+        return all([self._is_stream_done(stream_name) for stream_name in self._stream_name_to_instance.keys()])
 
     def _is_stream_done(self, stream_name: str) -> bool:
-        return (
-            all([p.is_closed() for p in self._streams_to_partitions[stream_name]])
-            and stream_name not in self._streams_currently_generating_partitions
-        )
+        return stream_name in self._streams_done
 
     def _on_stream_is_done(self, stream_name: str) -> AirbyteMessage:
         self._logger.info(f"Read {self._record_counter[stream_name]} records from {stream_name} stream")
         self._logger.info(f"Marking stream {stream_name} as STOPPED")
         stream = self._stream_name_to_instance[stream_name]
         self._logger.info(f"Finished syncing {stream.name}")
+        self._streams_done.add(stream_name)
         return stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.COMPLETE)
 
     def _stop_streams(self) -> Iterable[AirbyteMessage]:
         self._thread_pool_manager.shutdown()
-        for stream_name, partitions in self._streams_to_partitions.items():
+        for stream_name in self._streams_to_running_partitions.keys():
             stream = self._stream_name_to_instance[stream_name]
-            if not all([p.is_closed() for p in partitions]):
+            if not self._is_stream_done(stream_name):
                 self._logger.info(f"Marking stream {stream.name} as STOPPED")
                 self._logger.info(f"Finished syncing {stream.name}")
                 yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.INCOMPLETE)

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/airbyte_cdk/sources/concurrent_source/concurrent_source.py

@@ -16,6 +16,7 @@ from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionE
 from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
 from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
+from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue
 from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel, QueueItem
 from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger, SliceLogger
 
@@ -82,7 +83,7 @@ class ConcurrentSource:
         if not stream_instances_to_read_from:
             return
 
-        queue: Queue[QueueItem] = Queue()
+        queue: ThrottledQueue = ThrottledQueue(Queue(), self._threadpool.get_throttler(), self._timeout_seconds)
         concurrent_stream_processor = ConcurrentReadProcessor(
             stream_instances_to_read_from,
             PartitionEnqueuer(queue),
@@ -112,10 +113,10 @@ class ConcurrentSource:
 
     def _consume_from_queue(
         self,
-        queue: Queue[QueueItem],
+        queue: ThrottledQueue,
         concurrent_stream_processor: ConcurrentReadProcessor,
     ) -> Iterable[AirbyteMessage]:
-        while airbyte_message_or_record_or_exception := queue.get(block=True, timeout=self._timeout_seconds):
+        while airbyte_message_or_record_or_exception := queue.get():
             yield from self._handle_item(
                 airbyte_message_or_record_or_exception,
                 concurrent_stream_processor,

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py

@@ -2,10 +2,11 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 import logging
-import time
 from concurrent.futures import Future, ThreadPoolExecutor
 from typing import Any, Callable, List
 
+from airbyte_cdk.sources.concurrent_source.throttler import Throttler
+
 
 class ThreadPoolManager:
     """
@@ -19,8 +20,8 @@ class ThreadPoolManager:
         self,
         threadpool: ThreadPoolExecutor,
         logger: logging.Logger,
-        max_concurrent_tasks: int = DEFAULT_MAX_QUEUE_SIZE,
         sleep_time: float = DEFAULT_SLEEP_TIME,
+        max_concurrent_tasks: int = DEFAULT_MAX_QUEUE_SIZE,
     ):
         """
         :param threadpool: The threadpool to use
@@ -31,23 +32,17 @@ class ThreadPoolManager:
         self._threadpool = threadpool
         self._logger = logger
         self._max_concurrent_tasks = max_concurrent_tasks
-        self._sleep_time = sleep_time
         self._futures: List[Future[Any]] = []
+        self._throttler = Throttler(self._futures, sleep_time, max_concurrent_tasks)
+
+    def get_throttler(self) -> Throttler:
+        return self._throttler
 
     def submit(self, function: Callable[..., Any], *args: Any) -> None:
-        # Submit a task to the threadpool, waiting if there are too many pending tasks
-        self._wait_while_too_many_pending_futures(self._futures)
+        # Submit a task to the threadpool, removing completed tasks if there are too many tasks in self._futures.
+        self._prune_futures(self._futures)
         self._futures.append(self._threadpool.submit(function, *args))
 
-    def _wait_while_too_many_pending_futures(self, futures: List[Future[Any]]) -> None:
-        # Wait until the number of pending tasks is < self._max_concurrent_tasks
-        while True:
-            self._prune_futures(futures)
-            if len(futures) < self._max_concurrent_tasks:
-                break
-            self._logger.info("Main thread is sleeping because the task queue is full...")
-            time.sleep(self._sleep_time)
-
     def _prune_futures(self, futures: List[Future[Any]]) -> None:
         """
         Take a list in input and remove the futures that are completed. If a future has an exception, it'll raise and kill the stream
@@ -60,12 +55,14 @@ class ThreadPoolManager:
 
         for index in reversed(range(len(futures))):
             future = futures[index]
-            optional_exception = future.exception()
-            if optional_exception:
-                exception = RuntimeError(f"Failed reading with error: {optional_exception}")
-                self._stop_and_raise_exception(exception)
 
             if future.done():
+                # Only call future.exception() if the future is known to be done because it will block until the future is done.
+                # See https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Future.exception
+                optional_exception = future.exception()
+                if optional_exception:
+                    exception = RuntimeError(f"Failed reading with error: {optional_exception}")
+                    self._stop_and_raise_exception(exception)
                 futures.pop(index)
 
     def shutdown(self) -> None:

airbyte-cdk-0.59.0/airbyte_cdk/sources/concurrent_source/throttler.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+import time
+from concurrent.futures import Future
+from typing import Any, List
+
+
+class Throttler:
+    """
+    A throttler that waits until the number of concurrent tasks is below a certain threshold.
+    """
+
+    def __init__(self, futures_list: List[Future[Any]], sleep_time: float, max_concurrent_tasks: int):
+        """
+        :param futures_list: The list of futures to monitor
+        :param sleep_time: How long to sleep if there are too many pending tasks
+        :param max_concurrent_tasks: The maximum number of tasks that can be pending at the same time
+        """
+        self._futures_list = futures_list
+        self._sleep_time = sleep_time
+        self._max_concurrent_tasks = max_concurrent_tasks
+
+    def wait_and_acquire(self) -> None:
+        while len(self._futures_list) >= self._max_concurrent_tasks:
+            time.sleep(self._sleep_time)

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/airbyte_cdk/sources/streams/concurrent/cursor.py

@@ -3,7 +3,8 @@
 #
 import functools
 from abc import ABC, abstractmethod
-from typing import Any, List, Mapping, Optional, Protocol, Tuple
+from datetime import datetime
+from typing import Any, List, Mapping, MutableMapping, Optional, Protocol, Tuple
 
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.message import MessageRepository
@@ -36,6 +37,11 @@ class CursorField:
 
 
 class Cursor(ABC):
+    @property
+    @abstractmethod
+    def state(self) -> MutableMapping[str, Any]:
+        ...
+
     @abstractmethod
     def observe(self, record: Record) -> None:
         """
@@ -52,6 +58,10 @@ class Cursor(ABC):
 
 
 class NoopCursor(Cursor):
+    @property
+    def state(self) -> MutableMapping[str, Any]:
+        return {}
+
     def observe(self, record: Record) -> None:
         pass
 
@@ -73,6 +83,7 @@ class ConcurrentCursor(Cursor):
         connector_state_converter: AbstractStreamStateConverter,
         cursor_field: CursorField,
         slice_boundary_fields: Optional[Tuple[str, str]],
+        start: Optional[Any],
     ) -> None:
         self._stream_name = stream_name
         self._stream_namespace = stream_namespace
@@ -82,9 +93,19 @@ class ConcurrentCursor(Cursor):
         self._cursor_field = cursor_field
         # To see some example where the slice boundaries might not be defined, check https://github.com/airbytehq/airbyte/blob/1ce84d6396e446e1ac2377362446e3fb94509461/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py#L363-L379
         self._slice_boundary_fields = slice_boundary_fields if slice_boundary_fields else tuple()
+        self._start = start
         self._most_recent_record: Optional[Record] = None
         self._has_closed_at_least_one_slice = False
-        self.state = stream_state
+        self.start, self._concurrent_state = self._get_concurrent_state(stream_state)
+
+    @property
+    def state(self) -> MutableMapping[str, Any]:
+        return self._concurrent_state
+
+    def _get_concurrent_state(self, state: MutableMapping[str, Any]) -> Tuple[datetime, MutableMapping[str, Any]]:
+        if self._connector_state_converter.is_state_message_compatible(state):
+            return self._start or self._connector_state_converter.zero_value, self._connector_state_converter.deserialize(state)
+        return self._connector_state_converter.convert_from_sequential_state(self._cursor_field, state, self._start)
 
     def observe(self, record: Record) -> None:
         if self._slice_boundary_fields:
@@ -102,7 +123,7 @@ class ConcurrentCursor(Cursor):
     def close_partition(self, partition: Partition) -> None:
         slice_count_before = len(self.state.get("slices", []))
         self._add_slice_to_state(partition)
-        if slice_count_before < len(self.state["slices"]):
+        if slice_count_before < len(self.state["slices"]):  # only emit if at least one slice has been processed
             self._merge_partitions()
             self._emit_state_message()
         self._has_closed_at_least_one_slice = True
@@ -110,7 +131,9 @@ class ConcurrentCursor(Cursor):
     def _add_slice_to_state(self, partition: Partition) -> None:
         if self._slice_boundary_fields:
             if "slices" not in self.state:
-                self.state["slices"] = []
+                raise RuntimeError(
+                    f"The state for stream {self._stream_name} should have at least one slice to delineate the sync start time, but no slices are present. This is unexpected. Please contact Support."
+                )
             self.state["slices"].append(
                 {
                     "start": self._extract_from_slice(partition, self._slice_boundary_fields[self._START_BOUNDARY]),
@@ -126,10 +149,8 @@ class ConcurrentCursor(Cursor):
 
             self.state["slices"].append(
                 {
-                    # TODO: if we migrate stored state to the concurrent state format, we may want this to be the config start date
-                    #  instead of zero_value.
-                    "start": self._connector_state_converter.zero_value,
-                    "end": self._extract_cursor_value(self._most_recent_record),
+                    self._connector_state_converter.START_KEY: self.start,
+                    self._connector_state_converter.END_KEY: self._extract_cursor_value(self._most_recent_record),
                 }
             )
 

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py

@@ -2,11 +2,9 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from queue import Queue
-
 from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel
 from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
-from airbyte_cdk.sources.streams.concurrent.partitions.types import QueueItem
+from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue
 
 
 class PartitionEnqueuer:
@@ -14,10 +12,10 @@ class PartitionEnqueuer:
     Generates partitions from a partition generator and puts them in a queue.
     """
 
-    def __init__(self, queue: Queue[QueueItem]) -> None:
+    def __init__(self, queue: ThrottledQueue) -> None:
         """
         :param queue:  The queue to put the partitions in.
-        :param sentinel: The sentinel to put in the queue when all the partitions have been generated.
+        :param throttler: The throttler to use to throttle the partition generation.
         """
         self._queue = queue
 

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/airbyte_cdk/sources/streams/concurrent/partition_reader.py

@@ -2,10 +2,9 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from queue import Queue
-
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
-from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel, QueueItem
+from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue
+from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel
 
 
 class PartitionReader:
@@ -13,7 +12,7 @@ class PartitionReader:
     Generates records from a partition and puts them in a queue.
     """
 
-    def __init__(self, queue: Queue[QueueItem]) -> None:
+    def __init__(self, queue: ThrottledQueue) -> None:
         """
         :param queue: The queue to put the records in.
         """

airbyte-cdk-0.59.0/airbyte_cdk/sources/streams/concurrent/partitions/throttled_queue.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+from queue import Queue
+
+from airbyte_cdk.sources.concurrent_source.throttler import Throttler
+from airbyte_cdk.sources.streams.concurrent.partitions.types import QueueItem
+
+
+class ThrottledQueue:
+    """
+    A queue that throttles the number of items that can be added to it.
+
+    We throttle the queue using custom logic instead of relying on the queue's max size
+    because the main thread can continuously dequeue before submitting a future.
+
+    Since the main thread doesn't wait, it'll be able to remove items from the queue even if the tasks should be throttled,
+    so the tasks won't wait.
+
+    This class solves this issue by checking if we should throttle the queue before adding an item to it.
+    An example implementation of a throttler would check if the number of pending futures is greater than a certain threshold.
+    """
+
+    def __init__(self, queue: Queue[QueueItem], throttler: Throttler, timeout: float) -> None:
+        """
+        :param queue: The queue to throttle
+        :param throttler: The throttler to use to throttle the queue
+        :param timeout: The timeout to use when getting items from the queue
+        """
+        self._queue = queue
+        self._throttler = throttler
+        self._timeout = timeout
+
+    def put(self, item: QueueItem) -> None:
+        self._throttler.wait_and_acquire()
+        self._queue.put(item)
+
+    def get(self) -> QueueItem:
+        return self._queue.get(block=True, timeout=self._timeout)
+
+    def empty(self) -> bool:
+        return self._queue.empty()

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py

@@ -4,7 +4,7 @@
 
 from abc import ABC, abstractmethod
 from enum import Enum
-from typing import TYPE_CHECKING, Any, List, MutableMapping, Optional
+from typing import TYPE_CHECKING, Any, List, MutableMapping, Tuple
 
 if TYPE_CHECKING:
     from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
@@ -18,15 +18,6 @@ class AbstractStreamStateConverter(ABC):
     START_KEY = "start"
     END_KEY = "end"
 
-    def get_concurrent_stream_state(
-        self, cursor_field: Optional["CursorField"], state: MutableMapping[str, Any]
-    ) -> Optional[MutableMapping[str, Any]]:
-        if not cursor_field:
-            return None
-        if self.is_state_message_compatible(state):
-            return self.deserialize(state)
-        return self.convert_from_sequential_state(cursor_field, state)
-
     @abstractmethod
     def deserialize(self, state: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
         """
@@ -40,8 +31,11 @@ class AbstractStreamStateConverter(ABC):
 
     @abstractmethod
     def convert_from_sequential_state(
-        self, cursor_field: "CursorField", stream_state: MutableMapping[str, Any]
-    ) -> MutableMapping[str, Any]:
+        self,
+        cursor_field: "CursorField",
+        stream_state: MutableMapping[str, Any],
+        start: Any,
+    ) -> Tuple[Any, MutableMapping[str, Any]]:
         """
         Convert the state message to the format required by the ConcurrentCursor.
 

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py

@@ -4,7 +4,7 @@
 
 from abc import abstractmethod
 from datetime import datetime, timedelta
-from typing import Any, List, MutableMapping, Optional
+from typing import Any, List, MutableMapping, Optional, Tuple
 
 import pendulum
 from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
@@ -16,9 +16,6 @@ from pendulum.datetime import DateTime
 
 
 class DateTimeStreamStateConverter(AbstractStreamStateConverter):
-    START_KEY = "start"
-    END_KEY = "end"
-
     @property
     @abstractmethod
     def _zero_value(self) -> Any:
@@ -62,7 +59,7 @@ class DateTimeStreamStateConverter(AbstractStreamStateConverter):
         for interval in sorted_intervals[1:]:
             last_end_time = merged_intervals[-1][self.END_KEY]
             current_start_time = interval[self.START_KEY]
-            if self.compare_intervals(last_end_time, current_start_time):
+            if self._compare_intervals(last_end_time, current_start_time):
                 merged_end_time = max(last_end_time, interval[self.END_KEY])
                 merged_intervals[-1][self.END_KEY] = merged_end_time
             else:
@@ -70,10 +67,12 @@ class DateTimeStreamStateConverter(AbstractStreamStateConverter):
 
         return merged_intervals
 
-    def compare_intervals(self, end_time: Any, start_time: Any) -> bool:
+    def _compare_intervals(self, end_time: Any, start_time: Any) -> bool:
         return bool(self.increment(end_time) >= start_time)
 
-    def convert_from_sequential_state(self, cursor_field: CursorField, stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+    def convert_from_sequential_state(
+        self, cursor_field: CursorField, stream_state: MutableMapping[str, Any], start: datetime
+    ) -> Tuple[datetime, MutableMapping[str, Any]]:
         """
         Convert the state message to the format required by the ConcurrentCursor.
 
@@ -82,28 +81,35 @@ class DateTimeStreamStateConverter(AbstractStreamStateConverter):
             "state_type": ConcurrencyCompatibleStateType.date_range.value,
             "metadata": { … },
             "slices": [
-                {starts: 0, end: "2021-01-18T21:18:20.000+00:00", finished_processing: true}]
+                {"start": "2021-01-18T21:18:20.000+00:00", "end": "2021-01-18T21:18:20.000+00:00"},
+            ]
         }
         """
+        sync_start = self._get_sync_start(cursor_field, stream_state, start)
         if self.is_state_message_compatible(stream_state):
-            return stream_state
-        if cursor_field.cursor_field_key in stream_state:
-            slices = [
-                {
-                    # TODO: if we migrate stored state to the concurrent state format, we may want this to be the config start date
-                    # instead of `zero_value`
-                    self.START_KEY: self.zero_value,
-                    self.END_KEY: self.parse_timestamp(stream_state[cursor_field.cursor_field_key]),
-                },
-            ]
-        else:
-            slices = []
-        return {
+            return sync_start, stream_state
+
+        # Create a slice to represent the records synced during prior syncs.
+        # The start and end are the same to avoid confusion as to whether the records for this slice
+        # were actually synced
+        slices = [{self.START_KEY: sync_start, self.END_KEY: sync_start}]
+
+        return sync_start, {
             "state_type": ConcurrencyCompatibleStateType.date_range.value,
             "slices": slices,
             "legacy": stream_state,
         }
 
+    def _get_sync_start(self, cursor_field: CursorField, stream_state: MutableMapping[str, Any], start: Optional[Any]) -> datetime:
+        sync_start = self.parse_timestamp(start) if start is not None else self.zero_value
+        prev_sync_low_water_mark = (
+            self.parse_timestamp(stream_state[cursor_field.cursor_field_key]) if cursor_field.cursor_field_key in stream_state else None
+        )
+        if prev_sync_low_water_mark and prev_sync_low_water_mark >= sync_start:
+            return prev_sync_low_water_mark
+        else:
+            return sync_start
+
     def convert_to_sequential_state(self, cursor_field: CursorField, stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
         """
         Convert the state message from the concurrency-compatible format to the stream's original format.
@@ -113,10 +119,9 @@ class DateTimeStreamStateConverter(AbstractStreamStateConverter):
         """
         if self.is_state_message_compatible(stream_state):
             legacy_state = stream_state.get("legacy", {})
-            if slices := stream_state.pop("slices", None):
-                latest_complete_time = self._get_latest_complete_time(slices)
-                if latest_complete_time:
-                    legacy_state.update({cursor_field.cursor_field_key: self.output_format(latest_complete_time)})
+            latest_complete_time = self._get_latest_complete_time(stream_state.get("slices", []))
+            if latest_complete_time is not None:
+                legacy_state.update({cursor_field.cursor_field_key: self.output_format(latest_complete_time)})
             return legacy_state or {}
         else:
             return stream_state
@@ -125,11 +130,12 @@ class DateTimeStreamStateConverter(AbstractStreamStateConverter):
         """
         Get the latest time before which all records have been processed.
         """
-        if slices:
-            first_interval = self.merge_intervals(slices)[0][self.END_KEY]
-            return first_interval
-        else:
-            return None
+        if not slices:
+            raise RuntimeError("Expected at least one slice but there were none. This is unexpected; please contact Support.")
+
+        merged_intervals = self.merge_intervals(slices)
+        first_interval = merged_intervals[0]
+        return first_interval[self.END_KEY]
 
 
 class EpochValueConcurrentStreamStateConverter(DateTimeStreamStateConverter):

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.58.8
+Version: 0.59.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/airbyte_cdk.egg-info/SOURCES.txt

@@ -45,6 +45,7 @@ airbyte_cdk/sources/concurrent_source/concurrent_source.py
 airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py
 airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py
 airbyte_cdk/sources/concurrent_source/thread_pool_manager.py
+airbyte_cdk/sources/concurrent_source/throttler.py
 airbyte_cdk/sources/declarative/__init__.py
 airbyte_cdk/sources/declarative/create_partial.py
 airbyte_cdk/sources/declarative/declarative_component_schema.yaml
@@ -219,6 +220,7 @@ airbyte_cdk/sources/streams/concurrent/partitions/__init__.py
 airbyte_cdk/sources/streams/concurrent/partitions/partition.py
 airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py
 airbyte_cdk/sources/streams/concurrent/partitions/record.py
+airbyte_cdk/sources/streams/concurrent/partitions/throttled_queue.py
 airbyte_cdk/sources/streams/concurrent/partitions/types.py
 airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py
 airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py
@@ -423,6 +425,8 @@ unit_tests/sources/streams/concurrent/test_datetime_state_converter.py
 unit_tests/sources/streams/concurrent/test_default_stream.py
 unit_tests/sources/streams/concurrent/test_partition_reader.py
 unit_tests/sources/streams/concurrent/test_thread_pool_manager.py
+unit_tests/sources/streams/concurrent/test_throttled_queue.py
+unit_tests/sources/streams/concurrent/test_throttler.py
 unit_tests/sources/streams/concurrent/scenarios/__init__.py
 unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py
 unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/setup.py

@@ -36,7 +36,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.58.8",
+    version="0.59.0",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

{airbyte-cdk-0.58.8 → airbyte-cdk-0.59.0}/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py

@@ -52,8 +52,7 @@ class StreamFacadeSource(ConcurrentSourceAdapter):
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         state_manager = ConnectorStateManager(stream_instance_map={s.name: s for s in self._streams}, state=self._state)
         state_converter = StreamFacadeConcurrentConnectorStateConverter()
-        stream_states = [state_converter.get_concurrent_stream_state(self._cursor_field, state_manager.get_stream_state(stream.name, stream.namespace))
-                         for stream in self._streams]
+        stream_states = [state_manager.get_stream_state(stream.name, stream.namespace) for stream in self._streams]
         return [
             StreamFacade.create_from_stream(
                 stream,
@@ -69,6 +68,7 @@ class StreamFacadeSource(ConcurrentSourceAdapter):
                     state_converter,
                     self._cursor_field,
                     self._cursor_boundaries,
+                    None,
                 )
                 if self._cursor_field
                 else NoopCursor(),