airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py

@@ -0,0 +1,255 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+import logging
+from typing import Dict, Iterable, List, Optional, Set
+
+from airbyte_cdk.exception_handler import generate_failed_streams_error_message
+from airbyte_cdk.models import AirbyteMessage, AirbyteStreamStatus, FailureType, StreamDescriptor
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import (
+    PartitionGenerationCompletedSentinel,
+)
+from airbyte_cdk.sources.concurrent_source.stream_thread_exception import StreamThreadException
+from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager
+from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
+from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer
+from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader
+from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
+from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel
+from airbyte_cdk.sources.types import Record
+from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
+from airbyte_cdk.sources.utils.slice_logger import SliceLogger
+from airbyte_cdk.utils import AirbyteTracedException
+from airbyte_cdk.utils.stream_status_utils import (
+    as_airbyte_message as stream_status_as_airbyte_message,
+)
+
+
+class ConcurrentReadProcessor:
+    def __init__(
+        self,
+        stream_instances_to_read_from: List[AbstractStream],
+        partition_enqueuer: PartitionEnqueuer,
+        thread_pool_manager: ThreadPoolManager,
+        logger: logging.Logger,
+        slice_logger: SliceLogger,
+        message_repository: MessageRepository,
+        partition_reader: PartitionReader,
+    ):
+        """
+        This class is responsible for handling items from a concurrent stream read process.
+        :param stream_instances_to_read_from: List of streams to read from
+        :param partition_enqueuer: PartitionEnqueuer instance
+        :param thread_pool_manager: ThreadPoolManager instance
+        :param logger: Logger instance
+        :param slice_logger: SliceLogger instance
+        :param message_repository: MessageRepository instance
+        :param partition_reader: PartitionReader instance
+        """
+        self._stream_name_to_instance = {s.name: s for s in stream_instances_to_read_from}
+        self._record_counter = {}
+        self._streams_to_running_partitions: Dict[str, Set[Partition]] = {}
+        for stream in stream_instances_to_read_from:
+            self._streams_to_running_partitions[stream.name] = set()
+            self._record_counter[stream.name] = 0
+        self._thread_pool_manager = thread_pool_manager
+        self._partition_enqueuer = partition_enqueuer
+        self._stream_instances_to_start_partition_generation = stream_instances_to_read_from
+        self._streams_currently_generating_partitions: List[str] = []
+        self._logger = logger
+        self._slice_logger = slice_logger
+        self._message_repository = message_repository
+        self._partition_reader = partition_reader
+        self._streams_done: Set[str] = set()
+        self._exceptions_per_stream_name: dict[str, List[Exception]] = {}
+
+    def on_partition_generation_completed(
+        self, sentinel: PartitionGenerationCompletedSentinel
+    ) -> Iterable[AirbyteMessage]:
+        """
+        This method is called when a partition generation is completed.
+        1. Remove the stream from the list of streams currently generating partitions
+        2. If the stream is done, mark it as such and return a stream status message
+        3. If there are more streams to read from, start the next partition generator
+        """
+        stream_name = sentinel.stream.name
+        self._streams_currently_generating_partitions.remove(sentinel.stream.name)
+        # It is possible for the stream to already be done if no partitions were generated
+        # If the partition generation process was completed and there are no partitions left to process, the stream is done
+        if (
+            self._is_stream_done(stream_name)
+            or len(self._streams_to_running_partitions[stream_name]) == 0
+        ):
+            yield from self._on_stream_is_done(stream_name)
+        if self._stream_instances_to_start_partition_generation:
+            yield self.start_next_partition_generator()  # type:ignore # None may be yielded
+
+    def on_partition(self, partition: Partition) -> None:
+        """
+        This method is called when a partition is generated.
+        1. Add the partition to the set of partitions for the stream
+        2. Log the slice if necessary
+        3. Submit the partition to the thread pool manager
+        """
+        stream_name = partition.stream_name()
+        self._streams_to_running_partitions[stream_name].add(partition)
+        if self._slice_logger.should_log_slice_message(self._logger):
+            self._message_repository.emit_message(
+                self._slice_logger.create_slice_log_message(partition.to_slice())
+            )
+        self._thread_pool_manager.submit(self._partition_reader.process_partition, partition)
+
+    def on_partition_complete_sentinel(
+        self, sentinel: PartitionCompleteSentinel
+    ) -> Iterable[AirbyteMessage]:
+        """
+        This method is called when a partition is completed.
+        1. Close the partition
+        2. If the stream is done, mark it as such and return a stream status message
+        3. Emit messages that were added to the message repository
+        """
+        partition = sentinel.partition
+
+        try:
+            if sentinel.is_successful:
+                stream = self._stream_name_to_instance[partition.stream_name()]
+                stream.cursor.close_partition(partition)
+        except Exception as exception:
+            self._flag_exception(partition.stream_name(), exception)
+            yield AirbyteTracedException.from_exception(
+                exception, stream_descriptor=StreamDescriptor(name=partition.stream_name())
+            ).as_sanitized_airbyte_message()
+        finally:
+            partitions_running = self._streams_to_running_partitions[partition.stream_name()]
+            if partition in partitions_running:
+                partitions_running.remove(partition)
+                # If all partitions were generated and this was the last one, the stream is done
+                if (
+                    partition.stream_name() not in self._streams_currently_generating_partitions
+                    and len(partitions_running) == 0
+                ):
+                    yield from self._on_stream_is_done(partition.stream_name())
+            yield from self._message_repository.consume_queue()
+
+    def on_record(self, record: Record) -> Iterable[AirbyteMessage]:
+        """
+        This method is called when a record is read from a partition.
+        1. Convert the record to an AirbyteMessage
+        2. If this is the first record for the stream, mark the stream as RUNNING
+        3. Increment the record counter for the stream
+        4. Ensures the cursor knows the record has been successfully emitted
+        5. Emit the message
+        6. Emit messages that were added to the message repository
+        """
+        # Do not pass a transformer or a schema
+        # AbstractStreams are expected to return data as they are expected.
+        # Any transformation on the data should be done before reaching this point
+        message = stream_data_to_airbyte_message(
+            stream_name=record.stream_name,
+            data_or_message=record.data,
+            file_reference=record.file_reference,
+        )
+        stream = self._stream_name_to_instance[record.stream_name]
+
+        if message.type == MessageType.RECORD:
+            if self._record_counter[stream.name] == 0:
+                self._logger.info(f"Marking stream {stream.name} as RUNNING")
+                yield stream_status_as_airbyte_message(
+                    stream.as_airbyte_stream(), AirbyteStreamStatus.RUNNING
+                )
+            self._record_counter[stream.name] += 1
+            stream.cursor.observe(record)
+        yield message
+        yield from self._message_repository.consume_queue()
+
+    def on_exception(self, exception: StreamThreadException) -> Iterable[AirbyteMessage]:
+        """
+        This method is called when an exception is raised.
+        1. Stop all running streams
+        2. Raise the exception
+        """
+        self._flag_exception(exception.stream_name, exception.exception)
+        self._logger.exception(
+            f"Exception while syncing stream {exception.stream_name}", exc_info=exception.exception
+        )
+
+        stream_descriptor = StreamDescriptor(name=exception.stream_name)
+        if isinstance(exception.exception, AirbyteTracedException):
+            yield exception.exception.as_airbyte_message(stream_descriptor=stream_descriptor)
+        else:
+            yield AirbyteTracedException.from_exception(
+                exception, stream_descriptor=stream_descriptor
+            ).as_airbyte_message()
+
+    def _flag_exception(self, stream_name: str, exception: Exception) -> None:
+        self._exceptions_per_stream_name.setdefault(stream_name, []).append(exception)
+
+    def start_next_partition_generator(self) -> Optional[AirbyteMessage]:
+        """
+        Start the next partition generator.
+        1. Pop the next stream to read from
+        2. Submit the partition generator to the thread pool manager
+        3. Add the stream to the list of streams currently generating partitions
+        4. Return a stream status message
+        """
+        if self._stream_instances_to_start_partition_generation:
+            stream = self._stream_instances_to_start_partition_generation.pop(0)
+            self._thread_pool_manager.submit(self._partition_enqueuer.generate_partitions, stream)
+            self._streams_currently_generating_partitions.append(stream.name)
+            self._logger.info(f"Marking stream {stream.name} as STARTED")
+            self._logger.info(f"Syncing stream: {stream.name} ")
+            return stream_status_as_airbyte_message(
+                stream.as_airbyte_stream(),
+                AirbyteStreamStatus.STARTED,
+            )
+        else:
+            return None
+
+    def is_done(self) -> bool:
+        """
+        This method is called to check if the sync is done.
+        The sync is done when:
+        1. There are no more streams generating partitions
+        2. There are no more streams to read from
+        3. All partitions for all streams are closed
+        """
+        is_done = all(
+            [
+                self._is_stream_done(stream_name)
+                for stream_name in self._stream_name_to_instance.keys()
+            ]
+        )
+        if is_done and self._exceptions_per_stream_name:
+            error_message = generate_failed_streams_error_message(self._exceptions_per_stream_name)
+            self._logger.info(error_message)
+            # We still raise at least one exception when a stream raises an exception because the platform currently relies
+            # on a non-zero exit code to determine if a sync attempt has failed. We also raise the exception as a config_error
+            # type because this combined error isn't actionable, but rather the previously emitted individual errors.
+            raise AirbyteTracedException(
+                message=error_message,
+                internal_message="Concurrent read failure",
+                failure_type=FailureType.config_error,
+            )
+        return is_done
+
+    def _is_stream_done(self, stream_name: str) -> bool:
+        return stream_name in self._streams_done
+
+    def _on_stream_is_done(self, stream_name: str) -> Iterable[AirbyteMessage]:
+        self._logger.info(
+            f"Read {self._record_counter[stream_name]} records from {stream_name} stream"
+        )
+        self._logger.info(f"Marking stream {stream_name} as STOPPED")
+        stream = self._stream_name_to_instance[stream_name]
+        stream.cursor.ensure_at_least_one_state_emitted()
+        yield from self._message_repository.consume_queue()
+        self._logger.info(f"Finished syncing {stream.name}")
+        self._streams_done.add(stream_name)
+        stream_status = (
+            AirbyteStreamStatus.INCOMPLETE
+            if self._exceptions_per_stream_name.get(stream_name, [])
+            else AirbyteStreamStatus.COMPLETE
+        )
+        yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), stream_status)

airbyte_cdk/sources/concurrent_source/concurrent_source.py

@@ -0,0 +1,165 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+import concurrent
+import logging
+from queue import Queue
+from typing import Iterable, Iterator, List
+
+from airbyte_cdk.models import AirbyteMessage
+from airbyte_cdk.sources.concurrent_source.concurrent_read_processor import ConcurrentReadProcessor
+from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import (
+    PartitionGenerationCompletedSentinel,
+)
+from airbyte_cdk.sources.concurrent_source.stream_thread_exception import StreamThreadException
+from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager
+from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
+from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
+from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer
+from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader
+from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
+from airbyte_cdk.sources.streams.concurrent.partitions.types import (
+    PartitionCompleteSentinel,
+    QueueItem,
+)
+from airbyte_cdk.sources.types import Record
+from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger, SliceLogger
+
+
+class ConcurrentSource:
+    """
+    A Source that reads data from multiple AbstractStreams concurrently.
+    It does so by submitting partition generation, and partition read tasks to a thread pool.
+    The tasks asynchronously add their output to a shared queue.
+    The read is done when all partitions for all streams w ere generated and read.
+    """
+
+    DEFAULT_TIMEOUT_SECONDS = 900
+
+    @staticmethod
+    def create(
+        num_workers: int,
+        initial_number_of_partitions_to_generate: int,
+        logger: logging.Logger,
+        slice_logger: SliceLogger,
+        message_repository: MessageRepository,
+        timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
+    ) -> "ConcurrentSource":
+        is_single_threaded = initial_number_of_partitions_to_generate == 1 and num_workers == 1
+        too_many_generator = (
+            not is_single_threaded and initial_number_of_partitions_to_generate >= num_workers
+        )
+        assert (
+            not too_many_generator
+        ), "It is required to have more workers than threads generating partitions"
+        threadpool = ThreadPoolManager(
+            concurrent.futures.ThreadPoolExecutor(
+                max_workers=num_workers, thread_name_prefix="workerpool"
+            ),
+            logger,
+        )
+        return ConcurrentSource(
+            threadpool,
+            logger,
+            slice_logger,
+            message_repository,
+            initial_number_of_partitions_to_generate,
+            timeout_seconds,
+        )
+
+    def __init__(
+        self,
+        threadpool: ThreadPoolManager,
+        logger: logging.Logger,
+        slice_logger: SliceLogger = DebugSliceLogger(),
+        message_repository: MessageRepository = InMemoryMessageRepository(),
+        initial_number_partitions_to_generate: int = 1,
+        timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
+    ) -> None:
+        """
+        :param threadpool: The threadpool to submit tasks to
+        :param logger: The logger to log to
+        :param slice_logger: The slice logger used to create messages on new slices
+        :param message_repository: The repository to emit messages to
+        :param initial_number_partitions_to_generate: The initial number of concurrent partition generation tasks. Limiting this number ensures will limit the latency of the first records emitted. While the latency is not critical, emitting the records early allows the platform and the destination to process them as early as possible.
+        :param timeout_seconds: The maximum number of seconds to wait for a record to be read from the queue. If no record is read within this time, the source will stop reading and return.
+        """
+        self._threadpool = threadpool
+        self._logger = logger
+        self._slice_logger = slice_logger
+        self._message_repository = message_repository
+        self._initial_number_partitions_to_generate = initial_number_partitions_to_generate
+        self._timeout_seconds = timeout_seconds
+
+    def read(
+        self,
+        streams: List[AbstractStream],
+    ) -> Iterator[AirbyteMessage]:
+        self._logger.info("Starting syncing")
+
+        # We set a maxsize to for the main thread to process record items when the queue size grows. This assumes that there are less
+        # threads generating partitions that than are max number of workers. If it weren't the case, we could have threads only generating
+        # partitions which would fill the queue. This number is arbitrarily set to 10_000 but will probably need to be changed given more
+        # information and might even need to be configurable depending on the source
+        queue: Queue[QueueItem] = Queue(maxsize=10_000)
+        concurrent_stream_processor = ConcurrentReadProcessor(
+            streams,
+            PartitionEnqueuer(queue, self._threadpool),
+            self._threadpool,
+            self._logger,
+            self._slice_logger,
+            self._message_repository,
+            PartitionReader(queue),
+        )
+
+        # Enqueue initial partition generation tasks
+        yield from self._submit_initial_partition_generators(concurrent_stream_processor)
+
+        # Read from the queue until all partitions were generated and read
+        yield from self._consume_from_queue(
+            queue,
+            concurrent_stream_processor,
+        )
+        self._threadpool.check_for_errors_and_shutdown()
+        self._logger.info("Finished syncing")
+
+    def _submit_initial_partition_generators(
+        self, concurrent_stream_processor: ConcurrentReadProcessor
+    ) -> Iterable[AirbyteMessage]:
+        for _ in range(self._initial_number_partitions_to_generate):
+            status_message = concurrent_stream_processor.start_next_partition_generator()
+            if status_message:
+                yield status_message
+
+    def _consume_from_queue(
+        self,
+        queue: Queue[QueueItem],
+        concurrent_stream_processor: ConcurrentReadProcessor,
+    ) -> Iterable[AirbyteMessage]:
+        while airbyte_message_or_record_or_exception := queue.get():
+            yield from self._handle_item(
+                airbyte_message_or_record_or_exception,
+                concurrent_stream_processor,
+            )
+            if concurrent_stream_processor.is_done() and queue.empty():
+                # all partitions were generated and processed. we're done here
+                break
+
+    def _handle_item(
+        self,
+        queue_item: QueueItem,
+        concurrent_stream_processor: ConcurrentReadProcessor,
+    ) -> Iterable[AirbyteMessage]:
+        # handle queue item and call the appropriate handler depending on the type of the queue item
+        if isinstance(queue_item, StreamThreadException):
+            yield from concurrent_stream_processor.on_exception(queue_item)
+        elif isinstance(queue_item, PartitionGenerationCompletedSentinel):
+            yield from concurrent_stream_processor.on_partition_generation_completed(queue_item)
+        elif isinstance(queue_item, Partition):
+            concurrent_stream_processor.on_partition(queue_item)
+        elif isinstance(queue_item, PartitionCompleteSentinel):
+            yield from concurrent_stream_processor.on_partition_complete_sentinel(queue_item)
+        elif isinstance(queue_item, Record):
+            yield from concurrent_stream_processor.on_record(queue_item)
+        else:
+            raise ValueError(f"Unknown queue item type: {type(queue_item)}")

airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py

@@ -0,0 +1,147 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from abc import ABC
+from datetime import timedelta
+from typing import Any, Callable, Iterator, List, Mapping, MutableMapping, Optional, Tuple
+
+from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog
+from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource
+from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
+from airbyte_cdk.sources.streams.concurrent.abstract_stream_facade import AbstractStreamFacade
+from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade
+from airbyte_cdk.sources.streams.concurrent.cursor import (
+    ConcurrentCursor,
+    Cursor,
+    CursorField,
+    CursorValueType,
+    FinalStateCursor,
+    GapType,
+)
+from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import (
+    AbstractStreamStateConverter,
+)
+
+DEFAULT_LOOKBACK_SECONDS = 0
+
+
+class ConcurrentSourceAdapter(AbstractSource, ABC):
+    def __init__(self, concurrent_source: ConcurrentSource, **kwargs: Any) -> None:
+        """
+        ConcurrentSourceAdapter is a Source that wraps a concurrent source and exposes it as a regular source.
+
+        The source's streams are still defined through the streams() method.
+        Streams wrapped in a StreamFacade will be processed concurrently.
+        Other streams will be processed sequentially as a later step.
+        """
+        self._concurrent_source = concurrent_source
+        super().__init__(**kwargs)
+
+    def read(
+        self,
+        logger: logging.Logger,
+        config: Mapping[str, Any],
+        catalog: ConfiguredAirbyteCatalog,
+        state: Optional[List[AirbyteStateMessage]] = None,
+    ) -> Iterator[AirbyteMessage]:
+        abstract_streams = self._select_abstract_streams(config, catalog)
+        concurrent_stream_names = {stream.name for stream in abstract_streams}
+        configured_catalog_for_regular_streams = ConfiguredAirbyteCatalog(
+            streams=[
+                stream
+                for stream in catalog.streams
+                if stream.stream.name not in concurrent_stream_names
+            ]
+        )
+        if abstract_streams:
+            yield from self._concurrent_source.read(abstract_streams)
+        if configured_catalog_for_regular_streams.streams:
+            yield from super().read(logger, config, configured_catalog_for_regular_streams, state)
+
+    def _select_abstract_streams(
+        self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog
+    ) -> List[AbstractStream]:
+        """
+        Selects streams that can be processed concurrently and returns their abstract representations.
+        """
+        all_streams = self.streams(config)
+        stream_name_to_instance: Mapping[str, Stream] = {s.name: s for s in all_streams}
+        abstract_streams: List[AbstractStream] = []
+        for configured_stream in configured_catalog.streams:
+            stream_instance = stream_name_to_instance.get(configured_stream.stream.name)
+            if not stream_instance:
+                continue
+
+            if isinstance(stream_instance, AbstractStreamFacade):
+                abstract_streams.append(stream_instance.get_underlying_stream())
+        return abstract_streams
+
+    def convert_to_concurrent_stream(
+        self,
+        logger: logging.Logger,
+        stream: Stream,
+        state_manager: ConnectorStateManager,
+        cursor: Optional[Cursor] = None,
+    ) -> Stream:
+        """
+        Prepares a stream for concurrent processing by initializing or assigning a cursor,
+        managing the stream's state, and returning an updated Stream instance.
+        """
+        state: MutableMapping[str, Any] = {}
+
+        if cursor:
+            state = state_manager.get_stream_state(stream.name, stream.namespace)
+
+            stream.cursor = cursor  # type: ignore[assignment]  # cursor is of type ConcurrentCursor, which inherits from Cursor
+            if hasattr(stream, "parent"):
+                stream.parent.cursor = cursor
+        else:
+            cursor = FinalStateCursor(
+                stream_name=stream.name,
+                stream_namespace=stream.namespace,
+                message_repository=self.message_repository,  # type: ignore[arg-type]  # _default_message_repository will be returned in the worst case
+            )
+        return StreamFacade.create_from_stream(stream, self, logger, state, cursor)
+
+    def initialize_cursor(
+        self,
+        stream: Stream,
+        state_manager: ConnectorStateManager,
+        converter: AbstractStreamStateConverter,
+        slice_boundary_fields: Optional[Tuple[str, str]],
+        start: Optional[CursorValueType],
+        end_provider: Callable[[], CursorValueType],
+        lookback_window: Optional[GapType] = None,
+        slice_range: Optional[GapType] = None,
+    ) -> Optional[ConcurrentCursor]:
+        lookback_window = lookback_window or timedelta(seconds=DEFAULT_LOOKBACK_SECONDS)
+
+        cursor_field_name = stream.cursor_field
+
+        if cursor_field_name:
+            if not isinstance(cursor_field_name, str):
+                raise ValueError(
+                    f"Cursor field type must be a string, but received {type(cursor_field_name).__name__}."
+                )
+
+            return ConcurrentCursor(
+                stream.name,
+                stream.namespace,
+                state_manager.get_stream_state(stream.name, stream.namespace),
+                self.message_repository,  # type: ignore[arg-type]  # _default_message_repository will be returned in the worst case
+                state_manager,
+                converter,
+                CursorField(cursor_field_name),
+                slice_boundary_fields,
+                start,
+                end_provider,
+                lookback_window,
+                slice_range,
+            )
+
+        return None

airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py

@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+from typing import Any
+
+from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
+
+
+class PartitionGenerationCompletedSentinel:
+    """
+    A sentinel object indicating all partitions for a stream were produced.
+    Includes a pointer to the stream that was processed.
+    """
+
+    def __init__(self, stream: AbstractStream):
+        """
+        :param stream: The stream that was processed
+        """
+        self.stream = stream
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, PartitionGenerationCompletedSentinel):
+            return self.stream == other.stream
+        return False

airbyte_cdk/sources/concurrent_source/stream_thread_exception.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from typing import Any
+
+
+class StreamThreadException(Exception):
+    def __init__(self, exception: Exception, stream_name: str):
+        self._exception = exception
+        self._stream_name = stream_name
+
+    @property
+    def stream_name(self) -> str:
+        return self._stream_name
+
+    @property
+    def exception(self) -> Exception:
+        return self._exception
+
+    def __str__(self) -> str:
+        return f"Exception while syncing stream {self._stream_name}: {self._exception}"
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, StreamThreadException):
+            return self._exception == other._exception and self._stream_name == other._stream_name
+        return False