airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

airbyte_cdk/models/airbyte_protocol_serializers.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+from typing import Any, Dict
+
+from serpyco_rs import CustomType, Serializer
+
+from .airbyte_protocol import (  # type: ignore[attr-defined] # all classes are imported to airbyte_protocol via *
+    AirbyteMessage,
+    AirbyteStateBlob,
+    AirbyteStateMessage,
+    AirbyteStreamState,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    ConnectorSpecification,
+)
+
+
+class AirbyteStateBlobType(CustomType[AirbyteStateBlob, Dict[str, Any]]):
+    def serialize(self, value: AirbyteStateBlob) -> Dict[str, Any]:
+        # cant use orjson.dumps() directly because private attributes are excluded, e.g. "__ab_full_refresh_sync_complete"
+        return {k: v for k, v in value.__dict__.items()}
+
+    def deserialize(self, value: Dict[str, Any]) -> AirbyteStateBlob:
+        return AirbyteStateBlob(value)
+
+    def get_json_schema(self) -> Dict[str, Any]:
+        return {"type": "object"}
+
+
+def custom_type_resolver(t: type) -> CustomType[AirbyteStateBlob, Dict[str, Any]] | None:
+    return AirbyteStateBlobType() if t is AirbyteStateBlob else None
+
+
+AirbyteStreamStateSerializer = Serializer(
+    AirbyteStreamState, omit_none=True, custom_type_resolver=custom_type_resolver
+)
+AirbyteStateMessageSerializer = Serializer(
+    AirbyteStateMessage, omit_none=True, custom_type_resolver=custom_type_resolver
+)
+AirbyteMessageSerializer = Serializer(
+    AirbyteMessage, omit_none=True, custom_type_resolver=custom_type_resolver
+)
+ConfiguredAirbyteCatalogSerializer = Serializer(ConfiguredAirbyteCatalog, omit_none=True)
+ConfiguredAirbyteStreamSerializer = Serializer(ConfiguredAirbyteStream, omit_none=True)
+ConnectorSpecificationSerializer = Serializer(ConnectorSpecification, omit_none=True)

airbyte_cdk/models/file_transfer_record_message.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+
+@dataclass
+class AirbyteFileTransferRecordMessage:
+    stream: str
+    file: Dict[str, Any]
+    emitted_at: int
+    namespace: Optional[str] = None
+    data: Optional[Dict[str, Any]] = None

airbyte_cdk/models/well_known_types.py

@@ -2,4 +2,4 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_protocol.models.well_known_types import *
+from airbyte_protocol_dataclasses.models.well_known_types import *  # noqa: F403  # Allow '*'

airbyte_cdk/sources/__init__.py

@@ -19,4 +19,8 @@ from .source import Source
 # this will not be thread-safe.
 dpath.options.ALLOW_EMPTY_STRING_KEYS = True
 
-__all__ = ["AbstractSource", "BaseConfig", "Source"]
+__all__ = [
+    "AbstractSource",
+    "BaseConfig",
+    "Source",
+]

airbyte_cdk/sources/abstract_source.py

@@ -4,8 +4,20 @@
 
 import logging
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
+from airbyte_cdk.exception_handler import generate_failed_streams_error_message
 from airbyte_cdk.models import (
     AirbyteCatalog,
     AirbyteConnectionStatus,
@@ -17,21 +29,21 @@ from airbyte_cdk.models import (
     FailureType,
     Status,
     StreamDescriptor,
-    SyncMode,
 )
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.source import Source
-from airbyte_cdk.sources.streams import FULL_REFRESH_SENTINEL_STATE_KEY, Stream
+from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.core import StreamData
 from airbyte_cdk.sources.streams.http.http import HttpStream
 from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
 from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, split_config
 from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger, SliceLogger
-from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
 from airbyte_cdk.utils.event_timing import create_timer
-from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message
+from airbyte_cdk.utils.stream_status_utils import (
+    as_airbyte_message as stream_status_as_airbyte_message,
+)
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 _default_message_repository = InMemoryMessageRepository()
@@ -44,7 +56,9 @@ class AbstractSource(Source, ABC):
     """
 
     @abstractmethod
-    def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
+    def check_connection(
+        self, logger: logging.Logger, config: Mapping[str, Any]
+    ) -> Tuple[bool, Optional[Any]]:
         """
         :param logger: source logger
         :param config: The user-provided configuration as specified by the source's spec.
@@ -68,11 +82,6 @@ class AbstractSource(Source, ABC):
     _stream_to_instance_map: Dict[str, Stream] = {}
     _slice_logger: SliceLogger = DebugSliceLogger()
 
-    @property
-    def name(self) -> str:
-        """Source name"""
-        return self.__class__.__name__
-
     def discover(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCatalog:
         """Implements the Discover operation from the Airbyte Specification.
         See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#discover.
@@ -94,7 +103,7 @@ class AbstractSource(Source, ABC):
         logger: logging.Logger,
         config: Mapping[str, Any],
         catalog: ConfiguredAirbyteCatalog,
-        state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None,
+        state: Optional[List[AirbyteStateMessage]] = None,
     ) -> Iterator[AirbyteMessage]:
         """Implements the Read operation from the Airbyte Specification. See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/."""
         logger.info(f"Starting syncing {self.name}")
@@ -102,7 +111,7 @@ class AbstractSource(Source, ABC):
         # TODO assert all streams exist in the connector
         # get the streams once in case the connector needs to make any queries to generate them
         stream_instances = {s.name: s for s in self.streams(config)}
-        state_manager = ConnectorStateManager(stream_instance_map={s.stream.name: s.stream for s in catalog.streams}, state=state)
+        state_manager = ConnectorStateManager(state=state)
         self._stream_to_instance_map = stream_instances
 
         stream_name_to_exception: MutableMapping[str, AirbyteTracedException] = {}
@@ -110,22 +119,36 @@ class AbstractSource(Source, ABC):
         with create_timer(self.name) as timer:
             for configured_stream in catalog.streams:
                 stream_instance = stream_instances.get(configured_stream.stream.name)
-                if not stream_instance:
-                    if not self.raise_exception_on_missing_stream:
-                        continue
-                    raise KeyError(
-                        f"The stream {configured_stream.stream.name} no longer exists in the configuration. "
-                        f"Refresh the schema in replication settings and remove this stream from future sync attempts."
-                    )
-
+                is_stream_exist = bool(stream_instance)
                 try:
+                    # Used direct reference to `stream_instance` instead of `is_stream_exist` to avoid mypy type checking errors
+                    if not stream_instance:
+                        if not self.raise_exception_on_missing_stream:
+                            yield stream_status_as_airbyte_message(
+                                configured_stream.stream, AirbyteStreamStatus.INCOMPLETE
+                            )
+                            continue
+
+                        error_message = (
+                            f"The stream '{configured_stream.stream.name}' in your connection configuration was not found in the source. "
+                            f"Refresh the schema in your replication settings and remove this stream from future sync attempts."
+                        )
+
+                        # Use configured_stream as stream_instance to support references in error handling.
+                        stream_instance = configured_stream.stream
+
+                        raise AirbyteTracedException(
+                            message="A stream listed in your configuration was not found in the source. Please check the logs for more "
+                            "details.",
+                            internal_message=error_message,
+                            failure_type=FailureType.config_error,
+                        )
+
                     timer.start_event(f"Syncing stream {configured_stream.stream.name}")
-                    stream_is_available, reason = stream_instance.check_availability(logger, self)
-                    if not stream_is_available:
-                        logger.warning(f"Skipped syncing stream '{stream_instance.name}' because it was unavailable. {reason}")
-                        continue
                     logger.info(f"Marking stream {configured_stream.stream.name} as STARTED")
-                    yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.STARTED)
+                    yield stream_status_as_airbyte_message(
+                        configured_stream.stream, AirbyteStreamStatus.STARTED
+                    )
                     yield from self._read_stream(
                         logger=logger,
                         stream_instance=stream_instance,
@@ -134,52 +157,73 @@ class AbstractSource(Source, ABC):
                         internal_config=internal_config,
                     )
                     logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
-                    yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.COMPLETE)
-                except AirbyteTracedException as e:
-                    logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}")
-                    logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
-                    yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.INCOMPLETE)
-                    yield e.as_sanitized_airbyte_message(stream_descriptor=StreamDescriptor(name=configured_stream.stream.name))
-                    stream_name_to_exception[stream_instance.name] = e
-                    if self.stop_sync_on_stream_failure:
-                        logger.info(
-                            f"Stopping sync on error from stream {configured_stream.stream.name} because {self.name} does not support continuing syncs on error."
-                        )
-                        break
+                    yield stream_status_as_airbyte_message(
+                        configured_stream.stream, AirbyteStreamStatus.COMPLETE
+                    )
+
                 except Exception as e:
                     yield from self._emit_queued_messages()
-                    logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}")
+                    logger.exception(
+                        f"Encountered an exception while reading stream {configured_stream.stream.name}"
+                    )
                     logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
-                    yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.INCOMPLETE)
-                    display_message = stream_instance.get_error_display_message(e)
-                    if display_message:
-                        traced_exception = AirbyteTracedException.from_exception(e, message=display_message)
+                    yield stream_status_as_airbyte_message(
+                        configured_stream.stream, AirbyteStreamStatus.INCOMPLETE
+                    )
+
+                    stream_descriptor = StreamDescriptor(name=configured_stream.stream.name)
+
+                    if isinstance(e, AirbyteTracedException):
+                        traced_exception = e
+                        info_message = f"Stopping sync on error from stream {configured_stream.stream.name} because {self.name} does not support continuing syncs on error."
                     else:
-                        traced_exception = AirbyteTracedException.from_exception(e)
+                        traced_exception = self._serialize_exception(
+                            stream_descriptor, e, stream_instance=stream_instance
+                        )
+                        info_message = f"{self.name} does not support continuing syncs on error from stream {configured_stream.stream.name}"
+
                     yield traced_exception.as_sanitized_airbyte_message(
-                        stream_descriptor=StreamDescriptor(name=configured_stream.stream.name)
+                        stream_descriptor=stream_descriptor
                     )
-                    stream_name_to_exception[stream_instance.name] = traced_exception
+                    stream_name_to_exception[stream_instance.name] = traced_exception  # type: ignore # use configured_stream if stream_instance is None
                     if self.stop_sync_on_stream_failure:
-                        logger.info(f"{self.name} does not support continuing syncs on error from stream {configured_stream.stream.name}")
+                        logger.info(info_message)
                         break
                 finally:
-                    timer.finish_event()
-                    logger.info(f"Finished syncing {configured_stream.stream.name}")
-                    logger.info(timer.report())
+                    # Finish read event only if the stream instance exists;
+                    # otherwise, there's no need as it never started
+                    if is_stream_exist:
+                        timer.finish_event()
+                        logger.info(f"Finished syncing {configured_stream.stream.name}")
+                        logger.info(timer.report())
 
         if len(stream_name_to_exception) > 0:
-            error_message = self._generate_failed_streams_error_message(stream_name_to_exception)
+            error_message = generate_failed_streams_error_message(
+                {key: [value] for key, value in stream_name_to_exception.items()}
+            )
             logger.info(error_message)
             # We still raise at least one exception when a stream raises an exception because the platform currently relies
             # on a non-zero exit code to determine if a sync attempt has failed. We also raise the exception as a config_error
             # type because this combined error isn't actionable, but rather the previously emitted individual errors.
-            raise AirbyteTracedException(message=error_message, failure_type=FailureType.config_error)
+            raise AirbyteTracedException(
+                message=error_message, failure_type=FailureType.config_error
+            )
         logger.info(f"Finished syncing {self.name}")
 
+    @staticmethod
+    def _serialize_exception(
+        stream_descriptor: StreamDescriptor, e: Exception, stream_instance: Optional[Stream] = None
+    ) -> AirbyteTracedException:
+        display_message = stream_instance.get_error_display_message(e) if stream_instance else None
+        if display_message:
+            return AirbyteTracedException.from_exception(
+                e, message=display_message, stream_descriptor=stream_descriptor
+            )
+        return AirbyteTracedException.from_exception(e, stream_descriptor=stream_descriptor)
+
     @property
     def raise_exception_on_missing_stream(self) -> bool:
-        return True
+        return False
 
     def _read_stream(
         self,
@@ -190,7 +234,9 @@ class AbstractSource(Source, ABC):
         internal_config: InternalConfig,
     ) -> Iterator[AirbyteMessage]:
         if internal_config.page_size and isinstance(stream_instance, HttpStream):
-            logger.info(f"Setting page size for {stream_instance.name} to {internal_config.page_size}")
+            logger.info(
+                f"Setting page size for {stream_instance.name} to {internal_config.page_size}"
+            )
             stream_instance.page_size = internal_config.page_size
         logger.debug(
             f"Syncing configured stream: {configured_stream.stream.name}",
@@ -203,15 +249,16 @@ class AbstractSource(Source, ABC):
         stream_instance.log_stream_sync_configuration()
 
         stream_name = configured_stream.stream.name
-        # The platform always passes stream state regardless of sync mode. We shouldn't need to consider this case within the
-        # connector, but right now we need to prevent accidental usage of the previous stream state
-        stream_state = (
-            state_manager.get_stream_state(stream_name, stream_instance.namespace)
-            if configured_stream.sync_mode == SyncMode.incremental
-            else {}
-        )
+        stream_state = state_manager.get_stream_state(stream_name, stream_instance.namespace)
 
-        if stream_state and "state" in dir(stream_instance) and not self._stream_state_is_full_refresh(stream_state):
+        # This is a hack. Existing full refresh streams that are converted into resumable full refresh need to discard
+        # the state because the terminal state for a full refresh sync is not compatible with substream resumable full
+        # refresh state. This is only required when running live traffic regression testing since the platform normally
+        # handles whether to pass state
+        if stream_state == {"__ab_no_cursor_state_message": True}:
+            stream_state = {}
+
+        if "state" in dir(stream_instance):
             stream_instance.state = stream_state  # type: ignore # we check that state in the dir(stream_instance)
             logger.info(f"Setting state of {self.name} stream to {stream_state}")
 
@@ -233,7 +280,9 @@ class AbstractSource(Source, ABC):
                 if record_counter == 1:
                     logger.info(f"Marking stream {stream_name} as RUNNING")
                     # If we just read the first record of the stream, emit the transition to the RUNNING state
-                    yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.RUNNING)
+                    yield stream_status_as_airbyte_message(
+                        configured_stream.stream, AirbyteStreamStatus.RUNNING
+                    )
             yield from self._emit_queued_messages()
             yield record
 
@@ -244,14 +293,22 @@ class AbstractSource(Source, ABC):
             yield from self.message_repository.consume_queue()
         return
 
-    def _get_message(self, record_data_or_message: Union[StreamData, AirbyteMessage], stream: Stream) -> AirbyteMessage:
+    def _get_message(
+        self, record_data_or_message: Union[StreamData, AirbyteMessage], stream: Stream
+    ) -> AirbyteMessage:
         """
         Converts the input to an AirbyteMessage if it is a StreamData. Returns the input as is if it is already an AirbyteMessage
         """
-        if isinstance(record_data_or_message, AirbyteMessage):
-            return record_data_or_message
-        else:
-            return stream_data_to_airbyte_message(stream.name, record_data_or_message, stream.transformer, stream.get_json_schema())
+        match record_data_or_message:
+            case AirbyteMessage():
+                return record_data_or_message
+            case _:
+                return stream_data_to_airbyte_message(
+                    stream.name,
+                    record_data_or_message,
+                    stream.transformer,
+                    stream.get_json_schema(),
+                )
 
     @property
     def message_repository(self) -> Union[None, MessageRepository]:
@@ -267,14 +324,3 @@ class AbstractSource(Source, ABC):
         on the first error seen and emit a single error trace message for that stream.
         """
         return False
-
-    @staticmethod
-    def _generate_failed_streams_error_message(stream_failures: Mapping[str, AirbyteTracedException]) -> str:
-        failures = ", ".join([f"{stream}: {filter_secrets(exception.__repr__())}" for stream, exception in stream_failures.items()])
-        return f"During the sync, the following streams did not sync successfully: {failures}"
-
-    @staticmethod
-    def _stream_state_is_full_refresh(stream_state: Mapping[str, Any]) -> bool:
-        # For full refresh syncs that don't have a suitable cursor value, we emit a state that contains a sentinel key.
-        # This key is never used by a connector and is needed during a read to skip assigning the incoming state.
-        return FULL_REFRESH_SENTINEL_STATE_KEY in stream_state

airbyte_cdk/sources/concurrent_source/__init__.py

@@ -1,3 +1,8 @@
-#
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
+"""The concurrent source model replaces the legacy Source model.
+
+The concurrent source model is a new way to build sources in the Airbyte CDK. It is designed to
+be more ergonomic and performant than the legacy Source model.
+
+To implement a source using the concurrent source model, check out the submodules in this package.
+"""

airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py

@@ -4,20 +4,27 @@
 import logging
 from typing import Dict, Iterable, List, Optional, Set
 
-from airbyte_cdk.models import AirbyteMessage, AirbyteStreamStatus
+from airbyte_cdk.exception_handler import generate_failed_streams_error_message
+from airbyte_cdk.models import AirbyteMessage, AirbyteStreamStatus, FailureType, StreamDescriptor
 from airbyte_cdk.models import Type as MessageType
-from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel
+from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import (
+    PartitionGenerationCompletedSentinel,
+)
+from airbyte_cdk.sources.concurrent_source.stream_thread_exception import StreamThreadException
 from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager
 from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
 from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer
 from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
-from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
 from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel
+from airbyte_cdk.sources.types import Record
 from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
 from airbyte_cdk.sources.utils.slice_logger import SliceLogger
-from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message
+from airbyte_cdk.utils import AirbyteTracedException
+from airbyte_cdk.utils.stream_status_utils import (
+    as_airbyte_message as stream_status_as_airbyte_message,
+)
 
 
 class ConcurrentReadProcessor:
@@ -56,8 +63,11 @@ class ConcurrentReadProcessor:
         self._message_repository = message_repository
         self._partition_reader = partition_reader
         self._streams_done: Set[str] = set()
+        self._exceptions_per_stream_name: dict[str, List[Exception]] = {}
 
-    def on_partition_generation_completed(self, sentinel: PartitionGenerationCompletedSentinel) -> Iterable[AirbyteMessage]:
+    def on_partition_generation_completed(
+        self, sentinel: PartitionGenerationCompletedSentinel
+    ) -> Iterable[AirbyteMessage]:
         """
         This method is called when a partition generation is completed.
         1. Remove the stream from the list of streams currently generating partitions
@@ -68,10 +78,13 @@ class ConcurrentReadProcessor:
         self._streams_currently_generating_partitions.remove(sentinel.stream.name)
         # It is possible for the stream to already be done if no partitions were generated
         # If the partition generation process was completed and there are no partitions left to process, the stream is done
-        if self._is_stream_done(stream_name) or len(self._streams_to_running_partitions[stream_name]) == 0:
+        if (
+            self._is_stream_done(stream_name)
+            or len(self._streams_to_running_partitions[stream_name]) == 0
+        ):
             yield from self._on_stream_is_done(stream_name)
         if self._stream_instances_to_start_partition_generation:
-            yield self.start_next_partition_generator()
+            yield self.start_next_partition_generator()  # type:ignore # None may be yielded
 
     def on_partition(self, partition: Partition) -> None:
         """
@@ -83,10 +96,14 @@ class ConcurrentReadProcessor:
         stream_name = partition.stream_name()
         self._streams_to_running_partitions[stream_name].add(partition)
         if self._slice_logger.should_log_slice_message(self._logger):
-            self._message_repository.emit_message(self._slice_logger.create_slice_log_message(partition.to_slice()))
+            self._message_repository.emit_message(
+                self._slice_logger.create_slice_log_message(partition.to_slice())
+            )
         self._thread_pool_manager.submit(self._partition_reader.process_partition, partition)
 
-    def on_partition_complete_sentinel(self, sentinel: PartitionCompleteSentinel) -> Iterable[AirbyteMessage]:
+    def on_partition_complete_sentinel(
+        self, sentinel: PartitionCompleteSentinel
+    ) -> Iterable[AirbyteMessage]:
         """
         This method is called when a partition is completed.
         1. Close the partition
@@ -94,14 +111,27 @@ class ConcurrentReadProcessor:
         3. Emit messages that were added to the message repository
         """
         partition = sentinel.partition
-        partition.close()
-        partitions_running = self._streams_to_running_partitions[partition.stream_name()]
-        if partition in partitions_running:
-            partitions_running.remove(partition)
-            # If all partitions were generated and this was the last one, the stream is done
-            if partition.stream_name() not in self._streams_currently_generating_partitions and len(partitions_running) == 0:
-                yield from self._on_stream_is_done(partition.stream_name())
-        yield from self._message_repository.consume_queue()
+
+        try:
+            if sentinel.is_successful:
+                stream = self._stream_name_to_instance[partition.stream_name()]
+                stream.cursor.close_partition(partition)
+        except Exception as exception:
+            self._flag_exception(partition.stream_name(), exception)
+            yield AirbyteTracedException.from_exception(
+                exception, stream_descriptor=StreamDescriptor(name=partition.stream_name())
+            ).as_sanitized_airbyte_message()
+        finally:
+            partitions_running = self._streams_to_running_partitions[partition.stream_name()]
+            if partition in partitions_running:
+                partitions_running.remove(partition)
+                # If all partitions were generated and this was the last one, the stream is done
+                if (
+                    partition.stream_name() not in self._streams_currently_generating_partitions
+                    and len(partitions_running) == 0
+                ):
+                    yield from self._on_stream_is_done(partition.stream_name())
+            yield from self._message_repository.consume_queue()
 
     def on_record(self, record: Record) -> Iterable[AirbyteMessage]:
         """
@@ -109,31 +139,52 @@ class ConcurrentReadProcessor:
         1. Convert the record to an AirbyteMessage
         2. If this is the first record for the stream, mark the stream as RUNNING
         3. Increment the record counter for the stream
-        4. Emit the message
-        5. Emit messages that were added to the message repository
+        4. Ensures the cursor knows the record has been successfully emitted
+        5. Emit the message
+        6. Emit messages that were added to the message repository
         """
         # Do not pass a transformer or a schema
         # AbstractStreams are expected to return data as they are expected.
         # Any transformation on the data should be done before reaching this point
-        message = stream_data_to_airbyte_message(record.stream_name, record.data)
+        message = stream_data_to_airbyte_message(
+            stream_name=record.stream_name,
+            data_or_message=record.data,
+            is_file_transfer_message=record.is_file_transfer_message,
+        )
         stream = self._stream_name_to_instance[record.stream_name]
 
         if message.type == MessageType.RECORD:
             if self._record_counter[stream.name] == 0:
                 self._logger.info(f"Marking stream {stream.name} as RUNNING")
-                yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.RUNNING)
+                yield stream_status_as_airbyte_message(
+                    stream.as_airbyte_stream(), AirbyteStreamStatus.RUNNING
+                )
             self._record_counter[stream.name] += 1
+            stream.cursor.observe(record)
         yield message
         yield from self._message_repository.consume_queue()
 
-    def on_exception(self, exception: Exception) -> Iterable[AirbyteMessage]:
+    def on_exception(self, exception: StreamThreadException) -> Iterable[AirbyteMessage]:
         """
         This method is called when an exception is raised.
         1. Stop all running streams
         2. Raise the exception
         """
-        yield from self._stop_streams()
-        raise exception
+        self._flag_exception(exception.stream_name, exception.exception)
+        self._logger.exception(
+            f"Exception while syncing stream {exception.stream_name}", exc_info=exception.exception
+        )
+
+        stream_descriptor = StreamDescriptor(name=exception.stream_name)
+        if isinstance(exception.exception, AirbyteTracedException):
+            yield exception.exception.as_airbyte_message(stream_descriptor=stream_descriptor)
+        else:
+            yield AirbyteTracedException.from_exception(
+                exception, stream_descriptor=stream_descriptor
+            ).as_airbyte_message()
+
+    def _flag_exception(self, stream_name: str, exception: Exception) -> None:
+        self._exceptions_per_stream_name.setdefault(stream_name, []).append(exception)
 
     def start_next_partition_generator(self) -> Optional[AirbyteMessage]:
         """
@@ -164,26 +215,41 @@ class ConcurrentReadProcessor:
         2. There are no more streams to read from
         3. All partitions for all streams are closed
         """
-        return all([self._is_stream_done(stream_name) for stream_name in self._stream_name_to_instance.keys()])
+        is_done = all(
+            [
+                self._is_stream_done(stream_name)
+                for stream_name in self._stream_name_to_instance.keys()
+            ]
+        )
+        if is_done and self._exceptions_per_stream_name:
+            error_message = generate_failed_streams_error_message(self._exceptions_per_stream_name)
+            self._logger.info(error_message)
+            # We still raise at least one exception when a stream raises an exception because the platform currently relies
+            # on a non-zero exit code to determine if a sync attempt has failed. We also raise the exception as a config_error
+            # type because this combined error isn't actionable, but rather the previously emitted individual errors.
+            raise AirbyteTracedException(
+                message=error_message,
+                internal_message="Concurrent read failure",
+                failure_type=FailureType.config_error,
+            )
+        return is_done
 
     def _is_stream_done(self, stream_name: str) -> bool:
         return stream_name in self._streams_done
 
     def _on_stream_is_done(self, stream_name: str) -> Iterable[AirbyteMessage]:
-        self._logger.info(f"Read {self._record_counter[stream_name]} records from {stream_name} stream")
+        self._logger.info(
+            f"Read {self._record_counter[stream_name]} records from {stream_name} stream"
+        )
         self._logger.info(f"Marking stream {stream_name} as STOPPED")
         stream = self._stream_name_to_instance[stream_name]
         stream.cursor.ensure_at_least_one_state_emitted()
         yield from self._message_repository.consume_queue()
         self._logger.info(f"Finished syncing {stream.name}")
         self._streams_done.add(stream_name)
-        yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.COMPLETE)
-
-    def _stop_streams(self) -> Iterable[AirbyteMessage]:
-        self._thread_pool_manager.shutdown()
-        for stream_name in self._streams_to_running_partitions.keys():
-            stream = self._stream_name_to_instance[stream_name]
-            if not self._is_stream_done(stream_name):
-                self._logger.info(f"Marking stream {stream.name} as STOPPED")
-                self._logger.info(f"Finished syncing {stream.name}")
-                yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.INCOMPLETE)
+        stream_status = (
+            AirbyteStreamStatus.INCOMPLETE
+            if self._exceptions_per_stream_name.get(stream_name, [])
+            else AirbyteStreamStatus.COMPLETE
+        )
+        yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), stream_status)