airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/logger.py

@@ -0,0 +1,109 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import json
+import logging
+import logging.config
+from typing import Any, Callable, Mapping, Optional, Tuple
+
+import orjson
+
+from airbyte_cdk.models import (
+    AirbyteLogMessage,
+    AirbyteMessage,
+    AirbyteMessageSerializer,
+    Level,
+    Type,
+)
+from airbyte_cdk.utils import PrintBuffer
+from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
+
+PRINT_BUFFER = PrintBuffer(flush_interval=0.1)
+
+LOGGING_CONFIG = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "airbyte": {"()": "airbyte_cdk.logger.AirbyteLogFormatter", "format": "%(message)s"},
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "stream": PRINT_BUFFER,
+            "formatter": "airbyte",
+        },
+    },
+    "root": {
+        "handlers": ["console"],
+    },
+}
+
+
+def init_logger(name: Optional[str] = None) -> logging.Logger:
+    """Initial set up of logger"""
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+    logging.config.dictConfig(LOGGING_CONFIG)
+    return logger
+
+
+def lazy_log(logger: logging.Logger, level: int, lazy_log_provider: Callable[[], str]) -> None:
+    """
+    This method ensure that the processing of the log message is only done if the logger is enabled for the log level.
+    """
+    if logger.isEnabledFor(level):
+        logger.log(level, lazy_log_provider())
+
+
+class AirbyteLogFormatter(logging.Formatter):
+    """Output log records using AirbyteMessage"""
+
+    # Transforming Python log levels to Airbyte protocol log levels
+    level_mapping = {
+        logging.FATAL: Level.FATAL,
+        logging.ERROR: Level.ERROR,
+        logging.WARNING: Level.WARN,
+        logging.INFO: Level.INFO,
+        logging.DEBUG: Level.DEBUG,
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Return a JSON representation of the log message"""
+        airbyte_level = self.level_mapping.get(record.levelno, "INFO")
+        if airbyte_level == Level.DEBUG:
+            extras = self.extract_extra_args_from_record(record)
+            debug_dict = {"type": "DEBUG", "message": record.getMessage(), "data": extras}
+            return filter_secrets(json.dumps(debug_dict))
+        else:
+            message = super().format(record)
+            message = filter_secrets(message)
+            log_message = AirbyteMessage(
+                type=Type.LOG, log=AirbyteLogMessage(level=airbyte_level, message=message)
+            )
+            return orjson.dumps(AirbyteMessageSerializer.dump(log_message)).decode()
+
+    @staticmethod
+    def extract_extra_args_from_record(record: logging.LogRecord) -> Mapping[str, Any]:
+        """
+        The python logger conflates default args with extra args. We use an empty log record and set operations
+        to isolate fields passed to the log record via extra by the developer.
+        """
+        default_attrs = logging.LogRecord("", 0, "", 0, None, None, None).__dict__.keys()
+        extra_keys = set(record.__dict__.keys()) - default_attrs
+        return {k: str(getattr(record, k)) for k in extra_keys if hasattr(record, k)}
+
+
+def log_by_prefix(msg: str, default_level: str) -> Tuple[int, str]:
+    """Custom method, which takes log level from first word of message"""
+    valid_log_types = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"]
+    split_line = msg.split()
+    first_word = next(iter(split_line), None)
+    if first_word in valid_log_types:
+        log_level = logging.getLevelName(first_word)
+        rendered_message = " ".join(split_line[1:])
+    else:
+        log_level = logging.getLevelName(default_level)
+        rendered_message = msg
+
+    return log_level, rendered_message

airbyte_cdk/models/__init__.py

@@ -0,0 +1,72 @@
+# The earlier versions of airbyte-cdk (0.28.0<=) had the airbyte_protocol python classes
+# declared inline in the airbyte-cdk code. However, somewhere around Feb 2023 the
+# Airbyte Protocol moved to its own repo/PyPi package, called airbyte-protocol-models.
+# This directory including the airbyte_protocol.py and well_known_types.py files
+# are just wrappers on top of that stand-alone package which do some namespacing magic
+# to make the airbyte_protocol python classes available to the airbyte-cdk consumer as part
+# of airbyte-cdk rather than a standalone package.
+from .airbyte_protocol import (
+    AdvancedAuth,
+    AirbyteAnalyticsTraceMessage,
+    AirbyteCatalog,
+    AirbyteConnectionStatus,
+    AirbyteControlConnectorConfigMessage,
+    AirbyteControlMessage,
+    AirbyteErrorTraceMessage,
+    AirbyteEstimateTraceMessage,
+    AirbyteGlobalState,
+    AirbyteLogMessage,
+    AirbyteMessage,
+    AirbyteProtocol,
+    AirbyteRecordMessage,
+    AirbyteRecordMessageFileReference,
+    AirbyteStateBlob,
+    AirbyteStateMessage,
+    AirbyteStateStats,
+    AirbyteStateType,
+    AirbyteStream,
+    AirbyteStreamState,
+    AirbyteStreamStatus,
+    AirbyteStreamStatusReason,
+    AirbyteStreamStatusReasonType,
+    AirbyteStreamStatusTraceMessage,
+    AirbyteTraceMessage,
+    AuthFlowType,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    ConnectorSpecification,
+    DestinationSyncMode,
+    EstimateType,
+    FailureType,
+    Level,
+    OAuthConfigSpecification,
+    OauthConnectorInputSpecification,
+    OrchestratorType,
+    State,
+    Status,
+    StreamDescriptor,
+    SyncMode,
+    TraceType,
+    Type,
+)
+from .airbyte_protocol_serializers import (
+    AirbyteMessageSerializer,
+    AirbyteStateMessageSerializer,
+    AirbyteStreamStateSerializer,
+    ConfiguredAirbyteCatalogSerializer,
+    ConfiguredAirbyteStreamSerializer,
+    ConnectorSpecificationSerializer,
+)
+from .well_known_types import (
+    BinaryData,
+    Boolean,
+    Date,
+    Integer,
+    Model,
+    Number,
+    String,
+    TimestampWithoutTimezone,
+    TimestampWithTimezone,
+    TimeWithoutTimezone,
+    TimeWithTimezone,
+)

airbyte_cdk/models/airbyte_protocol.py

@@ -0,0 +1,88 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Annotated, Any, Dict, List, Mapping, Optional, Union
+
+from airbyte_protocol_dataclasses.models import *  # noqa: F403  # Allow '*'
+from serpyco_rs.metadata import Alias
+
+# ruff: noqa: F405  # ignore fuzzy import issues with 'import *'
+
+
+@dataclass
+class AirbyteStateBlob:
+    """
+    A dataclass that dynamically sets attributes based on provided keyword arguments and positional arguments.
+    Used to "mimic" pydantic Basemodel with ConfigDict(extra='allow') option.
+
+    The `AirbyteStateBlob` class allows for flexible instantiation by accepting any number of keyword arguments
+    and positional arguments. These are used to dynamically update the instance's attributes. This class is useful
+    in scenarios where the attributes of an object are not known until runtime and need to be set dynamically.
+
+    Attributes:
+        kwargs (InitVar[Mapping[str, Any]]): A dictionary of keyword arguments used to set attributes dynamically.
+
+    Methods:
+        __init__(*args: Any, **kwargs: Any) -> None:
+            Initializes the `AirbyteStateBlob` by setting attributes from the provided arguments.
+
+        __eq__(other: object) -> bool:
+            Checks equality between two `AirbyteStateBlob` instances based on their internal dictionaries.
+            Returns `False` if the other object is not an instance of `AirbyteStateBlob`.
+    """
+
+    kwargs: InitVar[Mapping[str, Any]]
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        # Set any attribute passed in through kwargs
+        for arg in args:
+            self.__dict__.update(arg)
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+    def __eq__(self, other: object) -> bool:
+        return (
+            False
+            if not isinstance(other, AirbyteStateBlob)
+            else bool(self.__dict__ == other.__dict__)
+        )
+
+
+# The following dataclasses have been redeclared to include the new version of AirbyteStateBlob
+@dataclass
+class AirbyteStreamState:
+    stream_descriptor: StreamDescriptor  # type: ignore [name-defined]
+    stream_state: Optional[AirbyteStateBlob] = None
+
+
+@dataclass
+class AirbyteGlobalState:
+    stream_states: List[AirbyteStreamState]
+    shared_state: Optional[AirbyteStateBlob] = None
+
+
+@dataclass
+class AirbyteStateMessage:
+    type: Optional[AirbyteStateType] = None  # type: ignore [name-defined]
+    stream: Optional[AirbyteStreamState] = None
+    global_: Annotated[AirbyteGlobalState | None, Alias("global")] = (
+        None  # "global" is a reserved keyword in python ⇒ Alias is used for (de-)serialization
+    )
+    data: Optional[Dict[str, Any]] = None
+    sourceStats: Optional[AirbyteStateStats] = None  # type: ignore [name-defined]
+    destinationStats: Optional[AirbyteStateStats] = None  # type: ignore [name-defined]
+
+
+@dataclass
+class AirbyteMessage:
+    type: Type  # type: ignore [name-defined]
+    log: Optional[AirbyteLogMessage] = None  # type: ignore [name-defined]
+    spec: Optional[ConnectorSpecification] = None  # type: ignore [name-defined]
+    connectionStatus: Optional[AirbyteConnectionStatus] = None  # type: ignore [name-defined]
+    catalog: Optional[AirbyteCatalog] = None  # type: ignore [name-defined]
+    record: Optional[AirbyteRecordMessage] = None  # type: ignore [name-defined]
+    state: Optional[AirbyteStateMessage] = None
+    trace: Optional[AirbyteTraceMessage] = None  # type: ignore [name-defined]
+    control: Optional[AirbyteControlMessage] = None  # type: ignore [name-defined]

airbyte_cdk/models/airbyte_protocol_serializers.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+from typing import Any, Dict
+
+from serpyco_rs import CustomType, Serializer
+
+from .airbyte_protocol import (  # type: ignore[attr-defined] # all classes are imported to airbyte_protocol via *
+    AirbyteMessage,
+    AirbyteStateBlob,
+    AirbyteStateMessage,
+    AirbyteStreamState,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    ConnectorSpecification,
+)
+
+
+class AirbyteStateBlobType(CustomType[AirbyteStateBlob, Dict[str, Any]]):
+    def serialize(self, value: AirbyteStateBlob) -> Dict[str, Any]:
+        # cant use orjson.dumps() directly because private attributes are excluded, e.g. "__ab_full_refresh_sync_complete"
+        return {k: v for k, v in value.__dict__.items()}
+
+    def deserialize(self, value: Dict[str, Any]) -> AirbyteStateBlob:
+        return AirbyteStateBlob(value)
+
+    def get_json_schema(self) -> Dict[str, Any]:
+        return {"type": "object"}
+
+
+def custom_type_resolver(t: type) -> CustomType[AirbyteStateBlob, Dict[str, Any]] | None:
+    return AirbyteStateBlobType() if t is AirbyteStateBlob else None
+
+
+AirbyteStreamStateSerializer = Serializer(
+    AirbyteStreamState, omit_none=True, custom_type_resolver=custom_type_resolver
+)
+AirbyteStateMessageSerializer = Serializer(
+    AirbyteStateMessage, omit_none=True, custom_type_resolver=custom_type_resolver
+)
+AirbyteMessageSerializer = Serializer(
+    AirbyteMessage, omit_none=True, custom_type_resolver=custom_type_resolver
+)
+ConfiguredAirbyteCatalogSerializer = Serializer(ConfiguredAirbyteCatalog, omit_none=True)
+ConfiguredAirbyteStreamSerializer = Serializer(ConfiguredAirbyteStream, omit_none=True)
+ConnectorSpecificationSerializer = Serializer(ConnectorSpecification, omit_none=True)

airbyte_cdk/models/well_known_types.py

@@ -0,0 +1,5 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_protocol_dataclasses.models.well_known_types import *  # noqa: F403  # Allow '*'

airbyte_cdk/sources/__init__.py

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+import dpath.options
+
+from .abstract_source import AbstractSource
+from .config import BaseConfig
+from .source import Source
+
+# As part of the CDK sources, we do not control what the APIs return and it is possible that a key is empty.
+# Reasons why we are doing this at the airbyte_cdk level:
+# * As of today, all the use cases should allow for empty keys
+#     * Cases as of 2023-08-31: oauth/session token provider responses, extractor, transformation and substream)
+# * The behavior is explicit at the package level and not hidden in every package that needs dpath.options.ALLOW_EMPTY_STRING_KEYS = True
+# There is a downside in enforcing this option preemptively in the module __init__.py: the runtime code will import dpath even though the it
+# might not need dpath leading to longer initialization time.
+# There is a downside in using dpath as a library since the options are global: if we have two pieces of code that want different options,
+# this will not be thread-safe.
+dpath.options.ALLOW_EMPTY_STRING_KEYS = True
+
+__all__ = [
+    "AbstractSource",
+    "BaseConfig",
+    "Source",
+]

airbyte_cdk/sources/abstract_source.py

@@ -0,0 +1,326 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from abc import ABC, abstractmethod
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Tuple,
+    Union,
+)
+
+from airbyte_cdk.exception_handler import generate_failed_streams_error_message
+from airbyte_cdk.models import (
+    AirbyteCatalog,
+    AirbyteConnectionStatus,
+    AirbyteMessage,
+    AirbyteStateMessage,
+    AirbyteStreamStatus,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    FailureType,
+    Status,
+    StreamDescriptor,
+)
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
+from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
+from airbyte_cdk.sources.source import Source
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.core import StreamData
+from airbyte_cdk.sources.streams.http.http import HttpStream
+from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
+from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, split_config
+from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger, SliceLogger
+from airbyte_cdk.utils.event_timing import create_timer
+from airbyte_cdk.utils.stream_status_utils import (
+    as_airbyte_message as stream_status_as_airbyte_message,
+)
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+
+_default_message_repository = InMemoryMessageRepository()
+
+
+class AbstractSource(Source, ABC):
+    """
+    Abstract base class for an Airbyte Source. Consumers should implement any abstract methods
+    in this class to create an Airbyte Specification compliant Source.
+    """
+
+    @abstractmethod
+    def check_connection(
+        self, logger: logging.Logger, config: Mapping[str, Any]
+    ) -> Tuple[bool, Optional[Any]]:
+        """
+        :param logger: source logger
+        :param config: The user-provided configuration as specified by the source's spec.
+          This usually contains information required to check connection e.g. tokens, secrets and keys etc.
+        :return: A tuple of (boolean, error). If boolean is true, then the connection check is successful
+          and we can connect to the underlying data source using the provided configuration.
+          Otherwise, the input config cannot be used to connect to the underlying data source,
+          and the "error" object should describe what went wrong.
+          The error object will be cast to string to display the problem to the user.
+        """
+
+    @abstractmethod
+    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
+        """
+        :param config: The user-provided configuration as specified by the source's spec.
+        Any stream construction related operation should happen here.
+        :return: A list of the streams in this source connector.
+        """
+
+    # Stream name to instance map for applying output object transformation
+    _stream_to_instance_map: Dict[str, Stream] = {}
+    _slice_logger: SliceLogger = DebugSliceLogger()
+
+    def discover(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCatalog:
+        """Implements the Discover operation from the Airbyte Specification.
+        See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#discover.
+        """
+        streams = [stream.as_airbyte_stream() for stream in self.streams(config=config)]
+        return AirbyteCatalog(streams=streams)
+
+    def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
+        """Implements the Check Connection operation from the Airbyte Specification.
+        See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#check.
+        """
+        check_succeeded, error = self.check_connection(logger, config)
+        if not check_succeeded:
+            return AirbyteConnectionStatus(status=Status.FAILED, message=repr(error))
+        return AirbyteConnectionStatus(status=Status.SUCCEEDED)
+
+    def read(
+        self,
+        logger: logging.Logger,
+        config: Mapping[str, Any],
+        catalog: ConfiguredAirbyteCatalog,
+        state: Optional[List[AirbyteStateMessage]] = None,
+    ) -> Iterator[AirbyteMessage]:
+        """Implements the Read operation from the Airbyte Specification. See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/."""
+        logger.info(f"Starting syncing {self.name}")
+        config, internal_config = split_config(config)
+        # TODO assert all streams exist in the connector
+        # get the streams once in case the connector needs to make any queries to generate them
+        stream_instances = {s.name: s for s in self.streams(config)}
+        state_manager = ConnectorStateManager(state=state)
+        self._stream_to_instance_map = stream_instances
+
+        stream_name_to_exception: MutableMapping[str, AirbyteTracedException] = {}
+
+        with create_timer(self.name) as timer:
+            for configured_stream in catalog.streams:
+                stream_instance = stream_instances.get(configured_stream.stream.name)
+                is_stream_exist = bool(stream_instance)
+                try:
+                    # Used direct reference to `stream_instance` instead of `is_stream_exist` to avoid mypy type checking errors
+                    if not stream_instance:
+                        if not self.raise_exception_on_missing_stream:
+                            yield stream_status_as_airbyte_message(
+                                configured_stream.stream, AirbyteStreamStatus.INCOMPLETE
+                            )
+                            continue
+
+                        error_message = (
+                            f"The stream '{configured_stream.stream.name}' in your connection configuration was not found in the source. "
+                            f"Refresh the schema in your replication settings and remove this stream from future sync attempts."
+                        )
+
+                        # Use configured_stream as stream_instance to support references in error handling.
+                        stream_instance = configured_stream.stream
+
+                        raise AirbyteTracedException(
+                            message="A stream listed in your configuration was not found in the source. Please check the logs for more "
+                            "details.",
+                            internal_message=error_message,
+                            failure_type=FailureType.config_error,
+                        )
+
+                    timer.start_event(f"Syncing stream {configured_stream.stream.name}")
+                    logger.info(f"Marking stream {configured_stream.stream.name} as STARTED")
+                    yield stream_status_as_airbyte_message(
+                        configured_stream.stream, AirbyteStreamStatus.STARTED
+                    )
+                    yield from self._read_stream(
+                        logger=logger,
+                        stream_instance=stream_instance,
+                        configured_stream=configured_stream,
+                        state_manager=state_manager,
+                        internal_config=internal_config,
+                    )
+                    logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
+                    yield stream_status_as_airbyte_message(
+                        configured_stream.stream, AirbyteStreamStatus.COMPLETE
+                    )
+
+                except Exception as e:
+                    yield from self._emit_queued_messages()
+                    logger.exception(
+                        f"Encountered an exception while reading stream {configured_stream.stream.name}"
+                    )
+                    logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
+                    yield stream_status_as_airbyte_message(
+                        configured_stream.stream, AirbyteStreamStatus.INCOMPLETE
+                    )
+
+                    stream_descriptor = StreamDescriptor(name=configured_stream.stream.name)
+
+                    if isinstance(e, AirbyteTracedException):
+                        traced_exception = e
+                        info_message = f"Stopping sync on error from stream {configured_stream.stream.name} because {self.name} does not support continuing syncs on error."
+                    else:
+                        traced_exception = self._serialize_exception(
+                            stream_descriptor, e, stream_instance=stream_instance
+                        )
+                        info_message = f"{self.name} does not support continuing syncs on error from stream {configured_stream.stream.name}"
+
+                    yield traced_exception.as_sanitized_airbyte_message(
+                        stream_descriptor=stream_descriptor
+                    )
+                    stream_name_to_exception[stream_instance.name] = traced_exception  # type: ignore # use configured_stream if stream_instance is None
+                    if self.stop_sync_on_stream_failure:
+                        logger.info(info_message)
+                        break
+                finally:
+                    # Finish read event only if the stream instance exists;
+                    # otherwise, there's no need as it never started
+                    if is_stream_exist:
+                        timer.finish_event()
+                        logger.info(f"Finished syncing {configured_stream.stream.name}")
+                        logger.info(timer.report())
+
+        if len(stream_name_to_exception) > 0:
+            error_message = generate_failed_streams_error_message(
+                {key: [value] for key, value in stream_name_to_exception.items()}
+            )
+            logger.info(error_message)
+            # We still raise at least one exception when a stream raises an exception because the platform currently relies
+            # on a non-zero exit code to determine if a sync attempt has failed. We also raise the exception as a config_error
+            # type because this combined error isn't actionable, but rather the previously emitted individual errors.
+            raise AirbyteTracedException(
+                message=error_message, failure_type=FailureType.config_error
+            )
+        logger.info(f"Finished syncing {self.name}")
+
+    @staticmethod
+    def _serialize_exception(
+        stream_descriptor: StreamDescriptor, e: Exception, stream_instance: Optional[Stream] = None
+    ) -> AirbyteTracedException:
+        display_message = stream_instance.get_error_display_message(e) if stream_instance else None
+        if display_message:
+            return AirbyteTracedException.from_exception(
+                e, message=display_message, stream_descriptor=stream_descriptor
+            )
+        return AirbyteTracedException.from_exception(e, stream_descriptor=stream_descriptor)
+
+    @property
+    def raise_exception_on_missing_stream(self) -> bool:
+        return False
+
+    def _read_stream(
+        self,
+        logger: logging.Logger,
+        stream_instance: Stream,
+        configured_stream: ConfiguredAirbyteStream,
+        state_manager: ConnectorStateManager,
+        internal_config: InternalConfig,
+    ) -> Iterator[AirbyteMessage]:
+        if internal_config.page_size and isinstance(stream_instance, HttpStream):
+            logger.info(
+                f"Setting page size for {stream_instance.name} to {internal_config.page_size}"
+            )
+            stream_instance.page_size = internal_config.page_size
+        logger.debug(
+            f"Syncing configured stream: {configured_stream.stream.name}",
+            extra={
+                "sync_mode": configured_stream.sync_mode,
+                "primary_key": configured_stream.primary_key,
+                "cursor_field": configured_stream.cursor_field,
+            },
+        )
+        stream_instance.log_stream_sync_configuration()
+
+        stream_name = configured_stream.stream.name
+        stream_state = state_manager.get_stream_state(stream_name, stream_instance.namespace)
+
+        # This is a hack. Existing full refresh streams that are converted into resumable full refresh need to discard
+        # the state because the terminal state for a full refresh sync is not compatible with substream resumable full
+        # refresh state. This is only required when running live traffic regression testing since the platform normally
+        # handles whether to pass state
+        if stream_state == {"__ab_no_cursor_state_message": True}:
+            stream_state = {}
+
+        if "state" in dir(stream_instance):
+            stream_instance.state = stream_state  # type: ignore # we check that state in the dir(stream_instance)
+            logger.info(f"Setting state of {self.name} stream to {stream_state}")
+
+        record_iterator = stream_instance.read(
+            configured_stream,
+            logger,
+            self._slice_logger,
+            stream_state,
+            state_manager,
+            internal_config,
+        )
+
+        record_counter = 0
+        logger.info(f"Syncing stream: {stream_name} ")
+        for record_data_or_message in record_iterator:
+            record = self._get_message(record_data_or_message, stream_instance)
+            if record.type == MessageType.RECORD:
+                record_counter += 1
+                if record_counter == 1:
+                    logger.info(f"Marking stream {stream_name} as RUNNING")
+                    # If we just read the first record of the stream, emit the transition to the RUNNING state
+                    yield stream_status_as_airbyte_message(
+                        configured_stream.stream, AirbyteStreamStatus.RUNNING
+                    )
+            yield from self._emit_queued_messages()
+            yield record
+
+        logger.info(f"Read {record_counter} records from {stream_name} stream")
+
+    def _emit_queued_messages(self) -> Iterable[AirbyteMessage]:
+        if self.message_repository:
+            yield from self.message_repository.consume_queue()
+        return
+
+    def _get_message(
+        self, record_data_or_message: Union[StreamData, AirbyteMessage], stream: Stream
+    ) -> AirbyteMessage:
+        """
+        Converts the input to an AirbyteMessage if it is a StreamData. Returns the input as is if it is already an AirbyteMessage
+        """
+        match record_data_or_message:
+            case AirbyteMessage():
+                return record_data_or_message
+            case _:
+                return stream_data_to_airbyte_message(
+                    stream.name,
+                    record_data_or_message,
+                    stream.transformer,
+                    stream.get_json_schema(),
+                )
+
+    @property
+    def message_repository(self) -> Union[None, MessageRepository]:
+        return _default_message_repository
+
+    @property
+    def stop_sync_on_stream_failure(self) -> bool:
+        """
+        WARNING: This function is in-development which means it is subject to change. Use at your own risk.
+
+        By default, when a source encounters an exception while syncing a stream, it will emit an error trace message and then
+        continue syncing the next stream. This can be overwritten on a per-source basis so that the source will stop the sync
+        on the first error seen and emit a single error trace message for that stream.
+        """
+        return False

airbyte_cdk/sources/concurrent_source/__init__.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+"""The concurrent source model replaces the legacy Source model.
+
+The concurrent source model is a new way to build sources in the Airbyte CDK. It is designed to
+be more ergonomic and performant than the legacy Source model.
+
+To implement a source using the concurrent source model, check out the submodules in this package.
+"""