airbyte-cdk 0.35.4__py3-none-any.whl → 0.36.1__py3-none-any.whl

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -50,7 +50,9 @@ def create_source(config: Mapping[str, Any], limits: TestReadLimits) -> Manifest
         component_factory=ModelToComponentFactory(
             emit_connector_builder_messages=True,
             limit_pages_fetched_per_slice=limits.max_pages_per_slice,
-            limit_slices_fetched=limits.max_slices)
+            limit_slices_fetched=limits.max_slices,
+            disable_retries=True
+        )
     )
 
 

airbyte_cdk/sources/abstract_source.py

@@ -13,6 +13,7 @@ from airbyte_cdk.models import (
     AirbyteLogMessage,
     AirbyteMessage,
     AirbyteStateMessage,
+    AirbyteStreamStatus,
     ConfiguredAirbyteCatalog,
     ConfiguredAirbyteStream,
     Level,
@@ -28,6 +29,7 @@ from airbyte_cdk.sources.streams.http.http import HttpStream
 from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
 from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, split_config
 from airbyte_cdk.utils.event_timing import create_timer
+from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 
@@ -113,6 +115,8 @@ class AbstractSource(Source, ABC):
                     continue
                 try:
                     timer.start_event(f"Syncing stream {configured_stream.stream.name}")
+                    logger.info(f"Marking stream {configured_stream.stream.name} as STARTED")
+                    yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.STARTED)
                     yield from self._read_stream(
                         logger=logger,
                         stream_instance=stream_instance,
@@ -120,10 +124,15 @@ class AbstractSource(Source, ABC):
                         state_manager=state_manager,
                         internal_config=internal_config,
                     )
+                    logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
+                    yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.COMPLETE)
                 except AirbyteTracedException as e:
+                    yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE)
                     raise e
                 except Exception as e:
                     logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}")
+                    logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
+                    yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE)
                     display_message = stream_instance.get_error_display_message(e)
                     if display_message:
                         raise AirbyteTracedException.from_exception(e, message=display_message) from e
@@ -185,6 +194,10 @@ class AbstractSource(Source, ABC):
         for record in record_iterator:
             if record.type == MessageType.RECORD:
                 record_counter += 1
+                if record_counter == 1:
+                    logger.info(f"Marking stream {stream_name} as RUNNING")
+                    # If we just read the first record of the stream, emit the transition to the RUNNING state
+                    yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.RUNNING)
             yield record
 
         logger.info(f"Read {record_counter} records from {stream_name} stream")

airbyte_cdk/sources/declarative/declarative_stream.py

@@ -5,14 +5,14 @@
 from dataclasses import InitVar, dataclass, field
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteTraceMessage, SyncMode
+from airbyte_cdk.models import AirbyteMessage, SyncMode
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
 from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader
 from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.declarative.types import Config, StreamSlice
-from airbyte_cdk.sources.streams.core import Stream
+from airbyte_cdk.sources.streams.core import Stream, StreamData
 
 
 @dataclass
@@ -102,17 +102,27 @@ class DeclarativeStream(Stream):
 
     def _apply_transformations(
         self,
-        message_or_record_data: Union[AirbyteMessage, AirbyteLogMessage, AirbyteTraceMessage, Mapping[str, Any]],
+        message_or_record_data: StreamData,
         config: Config,
         stream_slice: StreamSlice,
     ):
-        # If the input is an AirbyteRecord, transform the record's data
-        # If the input is another type of Airbyte Message, return it as is
+        # If the input is an AirbyteMessage with a record, transform the record's data
+        # If the input is another type of AirbyteMessage, return it as is
         # If the input is a dict, transform it
-        if isinstance(message_or_record_data, AirbyteLogMessage) or isinstance(message_or_record_data, AirbyteTraceMessage):
-            return message_or_record_data
+        if isinstance(message_or_record_data, AirbyteMessage):
+            if message_or_record_data.record:
+                record = message_or_record_data.record.data
+            else:
+                return message_or_record_data
+        elif isinstance(message_or_record_data, dict):
+            record = message_or_record_data
+        else:
+            # Raise an error because this is unexpected and indicative of a typing problem in the CDK
+            raise ValueError(
+                f"Unexpected record type. Expected {StreamData}. Got {type(message_or_record_data)}. This is probably due to a bug in the CDK."
+            )
         for transformation in self.transformations:
-            transformation.transform(message_or_record_data, config=config, stream_state=self.state, stream_slice=stream_slice)
+            transformation.transform(record, config=config, stream_state=self.state, stream_slice=stream_slice)
 
         return message_or_record_data
 

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -112,12 +112,17 @@ DEFAULT_BACKOFF_STRATEGY = ExponentialBackoffStrategy
 
 class ModelToComponentFactory:
     def __init__(
-        self, limit_pages_fetched_per_slice: int = None, limit_slices_fetched: int = None, emit_connector_builder_messages: bool = False
+        self,
+        limit_pages_fetched_per_slice: int = None,
+        limit_slices_fetched: int = None,
+        emit_connector_builder_messages: bool = False,
+        disable_retries=False,
     ):
         self._init_mappings()
         self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice
         self._limit_slices_fetched = limit_slices_fetched
         self._emit_connector_builder_messages = emit_connector_builder_messages
+        self._disable_retries = disable_retries
 
     def _init_mappings(self):
         self.PYDANTIC_MODEL_TO_CONSTRUCTOR: [Type[BaseModel], Callable] = {
@@ -779,6 +784,7 @@ class ModelToComponentFactory:
                 config=config,
                 maximum_number_of_slices=self._limit_slices_fetched,
                 parameters=model.parameters,
+                disable_retries=self._disable_retries,
             )
         return SimpleRetriever(
             name=name,
@@ -789,6 +795,7 @@ class ModelToComponentFactory:
             stream_slicer=stream_slicer or SinglePartitionRouter(parameters={}),
             config=config,
             parameters=model.parameters,
+            disable_retries=self._disable_retries,
         )
 
     @staticmethod

airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -50,6 +50,8 @@ class SimpleRetriever(Retriever, HttpStream):
         parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation
     """
 
+    _DEFAULT_MAX_RETRY = 5
+
     requester: Requester
     record_selector: HttpSelector
     config: Config
@@ -61,6 +63,7 @@ class SimpleRetriever(Retriever, HttpStream):
     paginator: Optional[Paginator] = None
     stream_slicer: Optional[StreamSlicer] = SinglePartitionRouter(parameters={})
     emit_connector_builder_messages: bool = False
+    disable_retries: bool = False
 
     def __post_init__(self, parameters: Mapping[str, Any]):
         self.paginator = self.paginator or NoPagination(parameters=parameters)
@@ -95,6 +98,14 @@ class SimpleRetriever(Retriever, HttpStream):
         # never raise on http_errors because this overrides the error handler logic...
         return False
 
+    @property
+    def max_retries(self) -> Union[int, None]:
+        if self.disable_retries:
+            return 0
+        if hasattr(self.requester.error_handler, "max_retries"):
+            return self.requester.error_handler.max_retries
+        return self._DEFAULT_MAX_RETRY
+
     def should_retry(self, response: requests.Response) -> bool:
         """
         Specifies conditions for backoff based on the response from the server.

airbyte_cdk/sources/streams/core.py

@@ -11,7 +11,7 @@ from functools import lru_cache
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union
 
 import airbyte_cdk.sources.utils.casing as casing
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteStream, AirbyteTraceMessage, SyncMode
+from airbyte_cdk.models import AirbyteMessage, AirbyteStream, SyncMode
 
 # list of all possible HTTP methods which can be used for sending of request bodies
 from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
@@ -24,10 +24,8 @@ if typing.TYPE_CHECKING:
 
 # A stream's read method can return one of the following types:
 # Mapping[str, Any]: The content of an AirbyteRecordMessage
-# AirbyteRecordMessage: An AirbyteRecordMessage
-# AirbyteLogMessage: A log message
-# AirbyteTraceMessage: A trace message
-StreamData = Union[Mapping[str, Any], AirbyteLogMessage, AirbyteTraceMessage]
+# AirbyteMessage: An AirbyteMessage. Could be of any type
+StreamData = Union[Mapping[str, Any], AirbyteMessage]
 
 
 def package_name_from_class(cls: object) -> str:

airbyte_cdk/utils/stream_status_utils.py

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from datetime import datetime
+
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteStreamStatus,
+    AirbyteStreamStatusTraceMessage,
+    AirbyteTraceMessage,
+    ConfiguredAirbyteStream,
+    StreamDescriptor,
+    TraceType,
+)
+from airbyte_cdk.models import Type as MessageType
+
+
+def as_airbyte_message(stream: ConfiguredAirbyteStream, current_status: AirbyteStreamStatus) -> AirbyteMessage:
+    """
+    Builds an AirbyteStreamStatusTraceMessage for the provided stream
+    """
+
+    now_millis = datetime.now().timestamp() * 1000.0
+
+    trace_message = AirbyteTraceMessage(
+        type=TraceType.STREAM_STATUS,
+        emitted_at=now_millis,
+        stream_status=AirbyteStreamStatusTraceMessage(
+            stream_descriptor=StreamDescriptor(name=stream.stream.name, namespace=stream.stream.namespace),
+            status=current_status,
+        ),
+    )
+
+    return AirbyteMessage(type=MessageType.TRACE, trace=trace_message)

{airbyte_cdk-0.35.4.dist-info → airbyte_cdk-0.36.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.35.4
+Version: 0.36.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.8
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: airbyte-protocol-models (==1.0.0)
+Requires-Dist: airbyte-protocol-models (==0.3.6)
 Requires-Dist: backoff
 Requires-Dist: dpath (~=2.0.1)
 Requires-Dist: isodate (~=0.6.1)

{airbyte_cdk-0.35.4.dist-info → airbyte_cdk-0.36.1.dist-info}/RECORD

@@ -6,7 +6,7 @@ airbyte_cdk/exception_handler.py,sha256=CwkiPdZ1WMOr3CBkvKFyHiyLerXGRqBrVlB4p0OI
 airbyte_cdk/logger.py,sha256=4Mi2MEQi1uh59BP9Dxw_UEbZuxaJewqK_jvEU2b10nk,3985
 airbyte_cdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=ytc8rNiGRNgDyY86LKnplLdQw-8k0r_J_9Bs37jMSl8,5364
+airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=KYfc3yyR9hWoj06E1-j2Z0OZtjeWw3YxdRda5z-dnq4,5407
 airbyte_cdk/connector_builder/main.py,sha256=eTXmkL1GaK1SCaN4jnVLh0MBok0KvM17QzQQ6qpsTTw,2982
 airbyte_cdk/connector_builder/message_grouper.py,sha256=swO_B0FovZMEliyjeYZ7Y7DRnTTwzrexy7rmqk7VnGU,11206
 airbyte_cdk/connector_builder/models.py,sha256=y0PJ-LwJk3e1RzRmMfjQSBP9ENx_a0wBcWNCjlW72Ks,1832
@@ -16,7 +16,7 @@ airbyte_cdk/models/__init__.py,sha256=LPQcYdDPwrCXiBPe_jexO4UAcbovIb1V9tHB6I7Un3
 airbyte_cdk/models/airbyte_protocol.py,sha256=wKXV_4sCzmUyPndiW7HWAj_A6EDRJyk9cA88xvXGQN0,117
 airbyte_cdk/models/well_known_types.py,sha256=KKfNbow2gdLoC1Z4hcXy_JR8m_acsB2ol7gQuEgjobw,117
 airbyte_cdk/sources/__init__.py,sha256=4j6fLtoRCjcZnojpise4EMmQtV1RepBxoGTBgpz80JA,218
-airbyte_cdk/sources/abstract_source.py,sha256=TVcsu-zBYu6fZ1PBipjyrqtKktgOXfu7fqTuMlIHYAk,16081
+airbyte_cdk/sources/abstract_source.py,sha256=svXe29SUHKA6WJwKlQlyKkZax2ybaG0wSvIFzaibl24,17262
 airbyte_cdk/sources/config.py,sha256=PYsY7y2u3EUwxLiEb96JnuKwH_E8CuxKggsRO2ZPSRc,856
 airbyte_cdk/sources/connector_state_manager.py,sha256=_R-2QnMGimKL0t5aV4f6P1dgd--TB3abY5Seg1xddXk,10469
 airbyte_cdk/sources/source.py,sha256=N3vHZzdUsBETFsql-YpO-LcgjolT_jcnAuHBhGD6Hqk,4278
@@ -24,7 +24,7 @@ airbyte_cdk/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4G
 airbyte_cdk/sources/declarative/create_partial.py,sha256=sUJOwD8hBzW4pxw2XhYlSTMgl-WMc5WpP5Oq_jo3fHw,3371
 airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=DJIV9HgHfBgv4w8zg6DhLIqXjZe235UVTe79IT7O51Q,74761
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=U2As9PDKmcWDgbsWUo-RetJ9fxQOBlwntWZ0NOgs5Ac,1453
-airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dhd228ddbG2YIptSw688HSWq9Hhh8GKLRxZS0VAz6rw,6211
+airbyte_cdk/sources/declarative/declarative_stream.py,sha256=0iZSpypxt8bhO3Lmf3BpGRTO7Fp0Q2GI8m8xyJJUjeM,6580
 airbyte_cdk/sources/declarative/exceptions.py,sha256=kTPUA4I2NV4J6HDz-mKPGMrfuc592akJnOyYx38l_QM,176
 airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=vTbRNM8D9P_ChOu1GNvtNRt-PM2L9N5Y0pNRyfVFuZg,9759
 airbyte_cdk/sources/declarative/types.py,sha256=b_RJpL9TyAgxJIRYZx5BxpC39p-WccHKxbAqxWrn9oE,482
@@ -66,7 +66,7 @@ airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=y7_G5mM07zxT
 airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py,sha256=W8BcK4KOg4ifNXgsdeIoV4oneHjXBKcPHEZHIC4r-hM,3801
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=H23H3nURCxsvjq66Gn9naffp0HJ1fU03wLFu-5F0AhQ,7701
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=6ukHx0bBrCJm9rek1l_MEfS3U_gdJcM4pJRyifJEOp0,6412
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=2t3Ye8Rota6X-rf8TfPj_yLPGVFRT91-KuhuWcB8DTc,46378
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=AzSjGeLahAA1FWvzbjIOk4iLiwHcaeS-tUhjs7bnyBk,46588
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=27sOWhw2LBQs62HchURakHQ2M_mtnOatNgU6q8RUtpU,476
 airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=fa6VtTwSoIkDI3SBoRtVx79opVtJX80_gU9bt31lspc,4785
 airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py,sha256=Fi3ocNZZoYkr0uvRgwoVSqne6enxRvi8DOHrASVK2PQ,1851
@@ -105,7 +105,7 @@ airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_
 airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py,sha256=tXxM0OPop49gw_EJuhJR5vyayXcg_XORrJlp5X8KydU,2625
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=IiHXDeKtibRqeWcRUckmSiXfk--u-sFMw3APWK8PCGQ,339
 airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=LBxg5r2QBPY8TSyQ7FxeG-FjLYFCV3Kq5MQM4SkCbUw,2030
-airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=BOjmMdbUjLeQ2DQKw656QreEaPZoT_enERJOEMZyuWE,20715
+airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=Yc5c1NQ0Jpgy5isrA6ltWIUW-Y1p9ih0aUMXNJUve0k,21057
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=ul8L9S0-__AMEdbCLHBq-PMEeA928NVp8BB83BMotfU,517
 airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=t0ll098cIG2Wr1rq1rZ3QDZ9WnScUuqAh42YVoTRWrU,1794
 airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRsatRJq3R3BeyRR0wIoK3gcP1gcpVRQ_P5U,464
@@ -128,7 +128,7 @@ airbyte_cdk/sources/singer/singer_helpers.py,sha256=q1LmgjFxSnN-dobMy7nikUwcK-9F
 airbyte_cdk/sources/singer/source.py,sha256=3YY8UTOXmctvMVUnYmIegmL3_IxF55iGP_bc_s2MZdY,8530
 airbyte_cdk/sources/streams/__init__.py,sha256=XGrzYjIkqItvnMshsOUzYhi4lC4M9kFHhxG0oCAoAyE,176
 airbyte_cdk/sources/streams/availability_strategy.py,sha256=7BM0qLvXS0QrlKvnVkBEw4Cw8i7PCENCBLcIAcuD3nY,1007
-airbyte_cdk/sources/streams/core.py,sha256=2yN4jCVEY1nAEpHCsRIERjMqLn6XpgEfEioLpzXlRo8,11133
+airbyte_cdk/sources/streams/core.py,sha256=G0MxhVkrF1DAPoPgmRBd_wnhrI7S6ZAdtOG0h9N_zHU,11021
 airbyte_cdk/sources/streams/http/__init__.py,sha256=6hRmA0P_RhB7X54xQbtj2RTz1RGlP52AG9V4pPWLaEQ,261
 airbyte_cdk/sources/streams/http/availability_strategy.py,sha256=uYYjanBG0f-CIrUYn1SnMyJqHoQQHnTggeJiB0m4i-Y,6344
 airbyte_cdk/sources/streams/http/exceptions.py,sha256=OokLDI7W8hZvq9e15sL3em2AdwmzmcAl72Ms-i5l0Nw,1334
@@ -156,11 +156,12 @@ airbyte_cdk/utils/__init__.py,sha256=kFLcs2P-tbPyeVOJS9rOv1jZdnSpjG24ro0CHgt_CIk
 airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=q3aDl8T10ufGbeqnUPqbZLxQcHdkf2kDfQK_upWzBbI,2894
 airbyte_cdk/utils/event_timing.py,sha256=Hn5kCc9xGKLcV5EYpJCZwNiz9neKKu2WG8FJF_hy278,2377
 airbyte_cdk/utils/schema_inferrer.py,sha256=LQLOlraFksg7_sqpJNhy9pS_K42GVxG634ogM_P2s5E,2361
+airbyte_cdk/utils/stream_status_utils.py,sha256=X1Vy7BhglycjdIWpfKDfwJussNCxYffelKt6Utjx-qY,1005
 airbyte_cdk/utils/traced_exception.py,sha256=9G2sG9eYkvn6Aa7rMuUW_KIRszRaTc_xdnTQNDKyKGI,3216
 source_declarative_manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 source_declarative_manifest/main.py,sha256=HXzuRsRyhHwPrGU-hc4S7RrgoOoHImqkdfbmO2geBeE,1027
 unit_tests/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-unit_tests/connector_builder/test_connector_builder_handler.py,sha256=-thlNWE20J37xPqfH5z89kvz09CVSIzull82IV6mNXQ,26729
+unit_tests/connector_builder/test_connector_builder_handler.py,sha256=tFcMzGHO2eLmbVtRPqLI3TKIo88tOY_ECEuG1D939p0,26796
 unit_tests/connector_builder/test_message_grouper.py,sha256=m4Iod2AwgKaRTkH8ndAC7Uq276BdzSg7SGsj1a4Jbhg,24086
 unit_tests/connector_builder/utils.py,sha256=AAggdGWP-mNuWOZUHLAVIbjTeIcdPo-3pbMm5zdYpS0,796
 unit_tests/destinations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -169,15 +170,15 @@ unit_tests/singer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 unit_tests/singer/test_singer_helpers.py,sha256=pZV6VxJuK-3-FICNGmoGbokrA_zkaFZEd4rYZCVpSRU,1762
 unit_tests/singer/test_singer_source.py,sha256=edN_kv7dnYAdBveWdUYOs74ak0dK6p8uaX225h_ZILA,4442
 unit_tests/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unit_tests/sources/test_abstract_source.py,sha256=ED6OjhaM60YQVV6c6ycRqjwWN0hHfLk_7pyzlaWrqKk,39753
+unit_tests/sources/test_abstract_source.py,sha256=eHZjhfSN-fzqbvdZtGqa5FVwggoFDXxi5SBA1-LQi70,44194
 unit_tests/sources/test_config.py,sha256=gFXqU_6OjwHXkV4JHMqQUznxmvTWN8nAv0w0-FFpugc,2477
 unit_tests/sources/test_connector_state_manager.py,sha256=ynFxA63Cxe6t-wMMh9C6ByTlMAuk8W7H2FikDhnUEQ0,24264
-unit_tests/sources/test_source.py,sha256=xE8LZrvsIp-mbsZmLQMOu3PVC2RmtIQZxjdYbNF4CGg,24151
+unit_tests/sources/test_source.py,sha256=eVtU9Zuc9gBsg11Pb5xjDtyU0gVrbYqbZ4RmzPvDw_M,24695
 unit_tests/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 unit_tests/sources/declarative/external_component.py,sha256=lU2gL736bLEWtmrGm1B2k83RXt_3XkROimLIahZd5dg,293
 unit_tests/sources/declarative/test_create_partial.py,sha256=s_KIywQqt8RlauOCWNJVk3HC3KBTAtSwFTN6JVQgu80,2636
-unit_tests/sources/declarative/test_declarative_stream.py,sha256=o-KNEH1NXOQc3ZkThHlasMHk9dPD1pbWXtX7O1BvN9w,4815
-unit_tests/sources/declarative/test_manifest_declarative_source.py,sha256=mPv2BMmLbkoI1DgT08Dsy4ll6PfzG52hNeQaZ1I13As,58906
+unit_tests/sources/declarative/test_declarative_stream.py,sha256=3leJnZIYHiFq8XI4jb3TjPXTubGJmvNGzABt4c01EkQ,5436
+unit_tests/sources/declarative/test_manifest_declarative_source.py,sha256=GckUc3nepzZkD1UM24woHlYCVZb5DP4IAQC3IeMyZF0,58924
 unit_tests/sources/declarative/test_yaml_declarative_source.py,sha256=6HhsUFgB7ueN0yOUHWb4gpPYLng5jasxN_plvz3x37g,5097
 unit_tests/sources/declarative/auth/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 unit_tests/sources/declarative/auth/test_oauth.py,sha256=mqXE_mQBcM78-ZaDX5GCWFOkbXPCvYeCj81aKyPZ3D8,5204
@@ -203,7 +204,7 @@ unit_tests/sources/declarative/interpolation/test_macros.py,sha256=q6kNuNfsrpupm
 unit_tests/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py,sha256=5lHUFv2n32b6h5IRh65S7EfqPkP5-IrGE3VUxDoPflI,12483
 unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py,sha256=K3q9eyx-sJFQ8nGYjAgS7fxau4sX_FlNreEAjiCYOeE,5306
-unit_tests/sources/declarative/parsers/test_model_to_component_factory.py,sha256=jBRkAF9ovA6Dj6CD8ff7GbeU1sWlMydmYzIg1ExG_lk,58188
+unit_tests/sources/declarative/parsers/test_model_to_component_factory.py,sha256=OKYCRk4_gK7x_9tvqUq2Yd1OXdHdPQRJcBqUj06jomA,58950
 unit_tests/sources/declarative/parsers/testing_components.py,sha256=_yUijmYRM-yYHPGDB2JsfEiOuVrgexGW9QwHf1xxNW8,1326
 unit_tests/sources/declarative/partition_routers/__init__.py,sha256=O8MZg4Bv_DghdRy9BoJCPIqdV75VtiUrhEkExQgb2nE,61
 unit_tests/sources/declarative/partition_routers/test_list_partition_router.py,sha256=gyivHDJ7iA6dslrI886GQnT_if0BfGmLPfiviVGiSqo,5118
@@ -233,7 +234,7 @@ unit_tests/sources/declarative/requesters/paginators/test_request_option.py,sha2
 unit_tests/sources/declarative/requesters/request_options/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py,sha256=Hl7b59ix1OwCJ5c34wn83d3D_l1dccE_nXlXWUR7Zos,5607
 unit_tests/sources/declarative/retrievers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
-unit_tests/sources/declarative/retrievers/test_simple_retriever.py,sha256=LhTKUN975T1af6ZqoOMkrzDq0I6wS7DGgaF15j0XS7g,27834
+unit_tests/sources/declarative/retrievers/test_simple_retriever.py,sha256=EhaUAo5ElmWW8pDRCD8K_SNhN-ct0ix7f-3DGTFp8Ag,29002
 unit_tests/sources/declarative/schema/__init__.py,sha256=i-iWyCqXPVgY-4miy16FH8U06gW_1_49AVq_8S8rVWY,134
 unit_tests/sources/declarative/schema/test_default_schema_loader.py,sha256=cWOFJnT9fhcEU6XLHkoe3E83mCjWc8lEttT0PFcvAm8,1091
 unit_tests/sources/declarative/schema/test_inline_schema_loader.py,sha256=vDJauhZ8og8M9ZqKDbf12SSYSfhUZ0_LmH7zjJHCHwI,517
@@ -256,9 +257,10 @@ unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.p
 unit_tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/utils/test_schema_inferrer.py,sha256=ckl17GlNOZInqgxni7Z2A0bg_p6JDy0GVFAG8ph67pw,3288
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
+unit_tests/utils/test_stream_status_utils.py,sha256=NpV155JMXA6CG-2Zvofa14lItobyh3Onttc59X4m5DI,3382
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.35.4.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.35.4.dist-info/METADATA,sha256=K4Z7tiT6fVjZHvrnAW4WJ7bk-76ddy-2nx8zTKbboec,8902
-airbyte_cdk-0.35.4.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-airbyte_cdk-0.35.4.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.35.4.dist-info/RECORD,,
+airbyte_cdk-0.36.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.36.1.dist-info/METADATA,sha256=julG0JMgo00RRk6h3nZf_h-2-wVJffJ3e_fEEsV3dhQ,8902
+airbyte_cdk-0.36.1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+airbyte_cdk-0.36.1.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.36.1.dist-info/RECORD,,

unit_tests/connector_builder/test_connector_builder_handler.py

@@ -566,6 +566,7 @@ def test_create_source():
     assert isinstance(source, ManifestDeclarativeSource)
     assert source._constructor._limit_pages_fetched_per_slice == limits.max_pages_per_slice
     assert source._constructor._limit_slices_fetched == limits.max_slices
+    assert source.streams(config={})[0].retriever.max_retries == 0
 
 
 def request_log_message(request: dict) -> AirbyteMessage:

unit_tests/sources/declarative/parsers/test_model_to_component_factory.py

@@ -1360,3 +1360,29 @@ def test_simple_retriever_emit_log_messages():
     )
 
     assert isinstance(retriever, SimpleRetrieverTestReadDecorator)
+
+
+def test_ignore_retry():
+    requester_model = {
+        "type": "SimpleRetriever",
+        "record_selector": {
+            "type": "RecordSelector",
+            "extractor": {
+                "type": "DpathExtractor",
+                "field_path": [],
+            },
+        },
+        "requester": {"type": "HttpRequester", "name": "list", "url_base": "orange.com", "path": "/v1/api"},
+    }
+
+    connector_builder_factory = ModelToComponentFactory(disable_retries=True)
+    retriever = connector_builder_factory.create_component(
+        model_type=SimpleRetrieverModel,
+        component_definition=requester_model,
+        config={},
+        name="Test",
+        primary_key="id",
+        stream_slicer=None,
+    )
+
+    assert retriever.max_retries == 0

unit_tests/sources/declarative/retrievers/test_simple_retriever.py

@@ -256,6 +256,48 @@ def test_parse_response(test_name, status_code, response_status, len_expected_re
         assert len(records) == len_expected_records
 
 
+def test_max_retries_given_error_handler_has_max_retries():
+    requester = MagicMock()
+    requester.error_handler = MagicMock()
+    requester.error_handler.max_retries = 10
+    retriever = SimpleRetriever(
+        name="stream_name",
+        primary_key=primary_key,
+        requester=requester,
+        record_selector=MagicMock(),
+        parameters={},
+        config={}
+    )
+    assert retriever.max_retries == 10
+
+
+def test_max_retries_given_error_handler_without_max_retries():
+    requester = MagicMock()
+    requester.error_handler = MagicMock(spec=[u'without_max_retries_attribute'])
+    retriever = SimpleRetriever(
+        name="stream_name",
+        primary_key=primary_key,
+        requester=requester,
+        record_selector=MagicMock(),
+        parameters={},
+        config={}
+    )
+    assert retriever.max_retries == 5
+
+
+def test_max_retries_given_disable_retries():
+    retriever = SimpleRetriever(
+        name="stream_name",
+        primary_key=primary_key,
+        requester=MagicMock(),
+        record_selector=MagicMock(),
+        disable_retries=True,
+        parameters={},
+        config={}
+    )
+    assert retriever.max_retries == 0
+
+
 @pytest.mark.parametrize(
     "test_name, response_action, retry_in, expected_backoff_time",
     [

unit_tests/sources/declarative/test_declarative_stream.py

@@ -5,7 +5,16 @@
 from unittest import mock
 from unittest.mock import MagicMock, call
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteTraceMessage, Level, SyncMode, TraceType
+from airbyte_cdk.models import (
+    AirbyteLogMessage,
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    AirbyteTraceMessage,
+    Level,
+    SyncMode,
+    TraceType,
+    Type,
+)
 from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.transformations import AddFields, RecordTransformation
 from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
@@ -24,8 +33,8 @@ def test_declarative_stream():
     records = [
         {"pk": 1234, "field": "value"},
         {"pk": 4567, "field": "different_value"},
-        AirbyteLogMessage(level=Level.INFO, message="This is a log  message"),
-        AirbyteTraceMessage(type=TraceType.ERROR, emitted_at=12345),
+        AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="This is a log  message")),
+        AirbyteMessage(type=Type.TRACE, trace=AirbyteTraceMessage(type=TraceType.ERROR, emitted_at=12345)),
     ]
     stream_slices = [
         {"date": "2021-01-01"},
@@ -84,15 +93,17 @@ def test_declarative_stream_with_add_fields_transform():
     retriever_records = [
         {"pk": 1234, "field": "value"},
         {"pk": 4567, "field": "different_value"},
-        AirbyteLogMessage(level=Level.INFO, message="This is a log  message"),
-        AirbyteTraceMessage(type=TraceType.ERROR, emitted_at=12345),
+        AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(data={"pk": 1357, "field": "a_value"}, emitted_at=12344, stream="stream")),
+        AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="This is a log  message")),
+        AirbyteMessage(type=Type.TRACE, trace=AirbyteTraceMessage(type=TraceType.ERROR, emitted_at=12345)),
     ]
 
     expected_records = [
         {"pk": 1234, "field": "value", "added_key": "added_value"},
         {"pk": 4567, "field": "different_value", "added_key": "added_value"},
-        AirbyteLogMessage(level=Level.INFO, message="This is a log  message"),
-        AirbyteTraceMessage(type=TraceType.ERROR, emitted_at=12345),
+        AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(data={"pk": 1357, "field": "a_value", "added_key": "added_value"}, emitted_at=12344, stream="stream")),
+        AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="This is a log  message")),
+        AirbyteMessage(type=Type.TRACE, trace=AirbyteTraceMessage(type=TraceType.ERROR, emitted_at=12345)),
     ]
     stream_slices = [
         {"date": "2021-01-01"},

unit_tests/sources/declarative/test_manifest_declarative_source.py

@@ -1242,7 +1242,7 @@ def _create_page(response_body):
 def test_read_manifest_declarative_source(test_name, manifest, pages, expected_records, expected_calls):
     _stream_name = "Rates"
     with patch.object(HttpStream, "_fetch_next_page", side_effect=pages) as mock_http_stream:
-        output_data = [message.record.data for message in _run_read(manifest, _stream_name)]
+        output_data = [message.record.data for message in _run_read(manifest, _stream_name) if message.record]
         assert expected_records == output_data
         mock_http_stream.assert_has_calls(expected_calls)
 

unit_tests/sources/test_abstract_source.py

@@ -21,6 +21,9 @@ from airbyte_cdk.models import (
     AirbyteStateType,
     AirbyteStream,
     AirbyteStreamState,
+    AirbyteStreamStatus,
+    AirbyteStreamStatusTraceMessage,
+    AirbyteTraceMessage,
     ConfiguredAirbyteCatalog,
     ConfiguredAirbyteStream,
     DestinationSyncMode,
@@ -28,8 +31,10 @@ from airbyte_cdk.models import (
     Status,
     StreamDescriptor,
     SyncMode,
-    Type,
+    TraceType,
 )
+from airbyte_cdk.models import Type
+from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.streams import IncrementalMixin, Stream
@@ -250,6 +255,19 @@ def _as_records(stream: str, data: List[Dict[str, Any]]) -> List[AirbyteMessage]
     return [_as_record(stream, datum) for datum in data]
 
 
+def _as_stream_status(stream: str, status: AirbyteStreamStatus) -> AirbyteMessage:
+    trace_message = AirbyteTraceMessage(
+        emitted_at=datetime.datetime.now().timestamp() * 1000.0,
+        type=TraceType.STREAM_STATUS,
+        stream_status=AirbyteStreamStatusTraceMessage(
+            stream_descriptor=StreamDescriptor(name=stream),
+            status=status,
+        ),
+    )
+
+    return AirbyteMessage(type=MessageType.TRACE, trace=trace_message)
+
+
 def _as_state(state_data: Dict[str, Any], stream_name: str = "", per_stream_state: Dict[str, Any] = None):
     if per_stream_state:
         return AirbyteMessage(
@@ -277,6 +295,8 @@ def _fix_emitted_at(messages: List[AirbyteMessage]) -> List[AirbyteMessage]:
     for msg in messages:
         if msg.type == Type.RECORD and msg.record:
             msg.record.emitted_at = GLOBAL_EMITTED_AT
+        if msg.type == Type.TRACE and msg.trace:
+            msg.trace.emitted_at = GLOBAL_EMITTED_AT
     return messages
 
 
@@ -296,7 +316,17 @@ def test_valid_full_refresh_read_no_slices(mocker):
         ]
     )
 
-    expected = _as_records("s1", stream_output) + _as_records("s2", stream_output)
+    expected = _fix_emitted_at(
+        [
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
+            *_as_records("s1", stream_output),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
+            _as_stream_status("s2", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s2", AirbyteStreamStatus.RUNNING),
+            *_as_records("s2", stream_output),
+            _as_stream_status("s2", AirbyteStreamStatus.COMPLETE)
+        ])
     messages = _fix_emitted_at(list(src.read(logger, {}, catalog)))
 
     assert expected == messages
@@ -326,7 +356,17 @@ def test_valid_full_refresh_read_with_slices(mocker):
         ]
     )
 
-    expected = [*_as_records("s1", slices), *_as_records("s2", slices)]
+    expected = _fix_emitted_at(
+        [
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
+            *_as_records("s1", slices),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
+            _as_stream_status("s2", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s2", AirbyteStreamStatus.RUNNING),
+            *_as_records("s2", slices),
+            _as_stream_status("s2", AirbyteStreamStatus.COMPLETE)
+        ])
 
     messages = _fix_emitted_at(list(src.read(logger, {}, catalog)))
 
@@ -448,18 +488,24 @@ class TestIncrementalRead:
             ]
         )
 
-        expected = [
+        expected = _fix_emitted_at([
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
             _as_record("s1", stream_output[0]),
             _as_record("s1", stream_output[1]),
             _as_state({"s1": new_state_from_connector}, "s1", new_state_from_connector)
             if per_stream_enabled
             else _as_state({"s1": new_state_from_connector}),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
+            _as_stream_status("s2", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s2", AirbyteStreamStatus.RUNNING),
             _as_record("s2", stream_output[0]),
             _as_record("s2", stream_output[1]),
             _as_state({"s1": new_state_from_connector, "s2": new_state_from_connector}, "s2", new_state_from_connector)
             if per_stream_enabled
             else _as_state({"s1": new_state_from_connector, "s2": new_state_from_connector}),
-        ]
+            _as_stream_status("s2", AirbyteStreamStatus.COMPLETE),
+        ])
         messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state)))
 
         assert messages == expected
@@ -521,18 +567,24 @@ class TestIncrementalRead:
             ]
         )
 
-        expected = [
+        expected = _fix_emitted_at([
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
             _as_record("s1", stream_output[0]),
             _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
             _as_record("s1", stream_output[1]),
             _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
             _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
+            _as_stream_status("s2", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s2", AirbyteStreamStatus.RUNNING),
             _as_record("s2", stream_output[0]),
             _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
             _as_record("s2", stream_output[1]),
             _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
             _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
-        ]
+            _as_stream_status("s2", AirbyteStreamStatus.COMPLETE),
+        ])
         messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state)))
 
         assert expected == messages
@@ -582,12 +634,18 @@ class TestIncrementalRead:
             ]
         )
 
-        expected = [
+        expected = _fix_emitted_at([
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
             *_as_records("s1", stream_output),
             _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
+            _as_stream_status("s2", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s2", AirbyteStreamStatus.RUNNING),
             *_as_records("s2", stream_output),
             _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
-        ]
+            _as_stream_status("s2", AirbyteStreamStatus.COMPLETE),
+        ])
 
         messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state)))
 
@@ -658,20 +716,26 @@ class TestIncrementalRead:
             ]
         )
 
-        expected = [
+        expected = _fix_emitted_at([
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
             # stream 1 slice 1
             *_as_records("s1", stream_output),
             _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
             # stream 1 slice 2
             *_as_records("s1", stream_output),
             _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
+            _as_stream_status("s2", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s2", AirbyteStreamStatus.RUNNING),
             # stream 2 slice 1
             *_as_records("s2", stream_output),
             _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
             # stream 2 slice 2
             *_as_records("s2", stream_output),
             _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
-        ]
+            _as_stream_status("s2", AirbyteStreamStatus.COMPLETE),
+        ])
 
         messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state)))
 
@@ -753,10 +817,14 @@ class TestIncrementalRead:
             ]
         )
 
-        expected = [
+        expected = _fix_emitted_at([
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
             _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
+            _as_stream_status("s2", AirbyteStreamStatus.STARTED),
             _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
-        ]
+            _as_stream_status("s2", AirbyteStreamStatus.COMPLETE),
+        ])
 
         messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state)))
 
@@ -837,8 +905,10 @@ class TestIncrementalRead:
             ]
         )
 
-        expected = [
+        expected = _fix_emitted_at([
             # stream 1 slice 1
+            _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
             _as_record("s1", stream_output[0]),
             _as_record("s1", stream_output[1]),
             _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
@@ -850,7 +920,10 @@ class TestIncrementalRead:
             _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
             _as_record("s1", stream_output[2]),
             _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
+            _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
             # stream 2 slice 1
+            _as_stream_status("s2", AirbyteStreamStatus.STARTED),
+            _as_stream_status("s2", AirbyteStreamStatus.RUNNING),
             _as_record("s2", stream_output[0]),
             _as_record("s2", stream_output[1]),
             _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
@@ -862,7 +935,8 @@ class TestIncrementalRead:
             _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
             _as_record("s2", stream_output[2]),
             _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
-        ]
+            _as_stream_status("s2", AirbyteStreamStatus.COMPLETE),
+        ])
 
         messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state)))
 
@@ -942,6 +1016,8 @@ class TestIncrementalRead:
 
         expected = _fix_emitted_at(
             [
+                _as_stream_status("s1", AirbyteStreamStatus.STARTED),
+                _as_stream_status("s1", AirbyteStreamStatus.RUNNING),
                 # stream 1 slice 1
                 stream_data_to_airbyte_message("s1", stream_output[0]),
                 stream_data_to_airbyte_message("s1", stream_output[1]),
@@ -956,7 +1032,10 @@ class TestIncrementalRead:
                 _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
                 stream_data_to_airbyte_message("s1", stream_output[3]),
                 _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}),
+                _as_stream_status("s1", AirbyteStreamStatus.COMPLETE),
                 # stream 2 slice 1
+                _as_stream_status("s2", AirbyteStreamStatus.STARTED),
+                _as_stream_status("s2", AirbyteStreamStatus.RUNNING),
                 stream_data_to_airbyte_message("s2", stream_output[0]),
                 stream_data_to_airbyte_message("s2", stream_output[1]),
                 stream_data_to_airbyte_message("s2", stream_output[2]),
@@ -970,6 +1049,7 @@ class TestIncrementalRead:
                 _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
                 stream_data_to_airbyte_message("s2", stream_output[3]),
                 _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}),
+                _as_stream_status("s2", AirbyteStreamStatus.COMPLETE),
             ]
         )
 

unit_tests/sources/test_source.py

@@ -365,8 +365,8 @@ def test_internal_config(abstract_source, catalog):
     # Test with empty config
     logger = logging.getLogger(f"airbyte.{getattr(abstract_source, 'name', '')}")
     records = [r for r in abstract_source.read(logger=logger, config={}, catalog=catalog, state={})]
-    # 3 for http stream and 3 for non http stream
-    assert len(records) == 3 + 3
+    # 3 for http stream, 3 for non http stream and 3 for stream status messages for each stream (2x)
+    assert len(records) == 3 + 3 + 3 + 3
     assert http_stream.read_records.called
     assert non_http_stream.read_records.called
     # Make sure page_size havent been set
@@ -375,21 +375,21 @@ def test_internal_config(abstract_source, catalog):
     # Test with records limit set to 1
     internal_config = {"some_config": 100, "_limit": 1}
     records = [r for r in abstract_source.read(logger=logger, config=internal_config, catalog=catalog, state={})]
-    # 1 from http stream + 1 from non http stream
-    assert len(records) == 1 + 1
+    # 1 from http stream + 1 from non http stream and 3 for stream status messages for each stream (2x)
+    assert len(records) == 1 + 1 + 3 + 3
     assert "_limit" not in abstract_source.streams_config
     assert "some_config" in abstract_source.streams_config
     # Test with records limit set to number that exceeds expceted records
     internal_config = {"some_config": 100, "_limit": 20}
     records = [r for r in abstract_source.read(logger=logger, config=internal_config, catalog=catalog, state={})]
-    assert len(records) == 3 + 3
+    assert len(records) == 3 + 3 + 3 + 3
 
     # Check if page_size paramter is set to http instance only
     internal_config = {"some_config": 100, "_page_size": 2}
     records = [r for r in abstract_source.read(logger=logger, config=internal_config, catalog=catalog, state={})]
     assert "_page_size" not in abstract_source.streams_config
     assert "some_config" in abstract_source.streams_config
-    assert len(records) == 3 + 3
+    assert len(records) == 3 + 3 + 3 + 3
     assert http_stream.page_size == 2
     # Make sure page_size havent been set for non http streams
     assert not non_http_stream.page_size
@@ -402,6 +402,7 @@ def test_internal_config_limit(mocker, abstract_source, catalog):
     STREAM_LIMIT = 2
     SLICE_DEBUG_LOG_COUNT = 1
     FULL_RECORDS_NUMBER = 3
+    TRACE_STATUS_COUNT = 3
     streams = abstract_source.streams(None)
     http_stream = streams[0]
     http_stream.read_records.return_value = [{}] * FULL_RECORDS_NUMBER
@@ -409,7 +410,7 @@ def test_internal_config_limit(mocker, abstract_source, catalog):
 
     catalog.streams[0].sync_mode = SyncMode.full_refresh
     records = [r for r in abstract_source.read(logger=logger_mock, config=internal_config, catalog=catalog, state={})]
-    assert len(records) == STREAM_LIMIT + SLICE_DEBUG_LOG_COUNT
+    assert len(records) == STREAM_LIMIT + SLICE_DEBUG_LOG_COUNT + TRACE_STATUS_COUNT
     logger_info_args = [call[0][0] for call in logger_mock.info.call_args_list]
     # Check if log line matches number of limit
     read_log_record = [_l for _l in logger_info_args if _l.startswith("Read")]
@@ -418,14 +419,16 @@ def test_internal_config_limit(mocker, abstract_source, catalog):
     # No limit, check if state record produced for incremental stream
     catalog.streams[0].sync_mode = SyncMode.incremental
     records = [r for r in abstract_source.read(logger=logger_mock, config={}, catalog=catalog, state={})]
-    assert len(records) == FULL_RECORDS_NUMBER + SLICE_DEBUG_LOG_COUNT + 1
-    assert records[-1].type == Type.STATE
+    assert len(records) == FULL_RECORDS_NUMBER + SLICE_DEBUG_LOG_COUNT + TRACE_STATUS_COUNT + 1
+    assert records[-2].type == Type.STATE
+    assert records[-1].type == Type.TRACE
 
     # Set limit and check if state is produced when limit is set for incremental stream
     logger_mock.reset_mock()
     records = [r for r in abstract_source.read(logger=logger_mock, config=internal_config, catalog=catalog, state={})]
-    assert len(records) == STREAM_LIMIT + SLICE_DEBUG_LOG_COUNT + 1
-    assert records[-1].type == Type.STATE
+    assert len(records) == STREAM_LIMIT + SLICE_DEBUG_LOG_COUNT + TRACE_STATUS_COUNT + 1
+    assert records[-2].type == Type.STATE
+    assert records[-1].type == Type.TRACE
     logger_info_args = [call[0][0] for call in logger_mock.info.call_args_list]
     read_log_record = [_l for _l in logger_info_args if _l.startswith("Read")]
     assert read_log_record[0].startswith(f"Read {STREAM_LIMIT} ")
@@ -436,6 +439,7 @@ SCHEMA = {"type": "object", "properties": {"value": {"type": "string"}}}
 
 def test_source_config_no_transform(mocker, abstract_source, catalog):
     SLICE_DEBUG_LOG_COUNT = 1
+    TRACE_STATUS_COUNT = 3
     logger_mock = mocker.MagicMock()
     logger_mock.level = logging.DEBUG
     streams = abstract_source.streams(None)
@@ -443,7 +447,7 @@ def test_source_config_no_transform(mocker, abstract_source, catalog):
     http_stream.get_json_schema.return_value = non_http_stream.get_json_schema.return_value = SCHEMA
     http_stream.read_records.return_value, non_http_stream.read_records.return_value = [[{"value": 23}] * 5] * 2
     records = [r for r in abstract_source.read(logger=logger_mock, config={}, catalog=catalog, state={})]
-    assert len(records) == 2 * (5 + SLICE_DEBUG_LOG_COUNT)
+    assert len(records) == 2 * (5 + SLICE_DEBUG_LOG_COUNT + TRACE_STATUS_COUNT)
     assert [r.record.data for r in records if r.type == Type.RECORD] == [{"value": 23}] * 2 * 5
     assert http_stream.get_json_schema.call_count == 5
     assert non_http_stream.get_json_schema.call_count == 5
@@ -453,6 +457,7 @@ def test_source_config_transform(mocker, abstract_source, catalog):
     logger_mock = mocker.MagicMock()
     logger_mock.level = logging.DEBUG
     SLICE_DEBUG_LOG_COUNT = 2
+    TRACE_STATUS_COUNT = 6
     streams = abstract_source.streams(None)
     http_stream, non_http_stream = streams
     http_stream.transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
@@ -460,7 +465,7 @@ def test_source_config_transform(mocker, abstract_source, catalog):
     http_stream.get_json_schema.return_value = non_http_stream.get_json_schema.return_value = SCHEMA
     http_stream.read_records.return_value, non_http_stream.read_records.return_value = [{"value": 23}], [{"value": 23}]
     records = [r for r in abstract_source.read(logger=logger_mock, config={}, catalog=catalog, state={})]
-    assert len(records) == 2 + SLICE_DEBUG_LOG_COUNT
+    assert len(records) == 2 + SLICE_DEBUG_LOG_COUNT + TRACE_STATUS_COUNT
     assert [r.record.data for r in records if r.type == Type.RECORD] == [{"value": "23"}] * 2
 
 
@@ -468,13 +473,14 @@ def test_source_config_transform_and_no_transform(mocker, abstract_source, catal
     logger_mock = mocker.MagicMock()
     logger_mock.level = logging.DEBUG
     SLICE_DEBUG_LOG_COUNT = 2
+    TRACE_STATUS_COUNT = 6
     streams = abstract_source.streams(None)
     http_stream, non_http_stream = streams
     http_stream.transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
     http_stream.get_json_schema.return_value = non_http_stream.get_json_schema.return_value = SCHEMA
     http_stream.read_records.return_value, non_http_stream.read_records.return_value = [{"value": 23}], [{"value": 23}]
     records = [r for r in abstract_source.read(logger=logger_mock, config={}, catalog=catalog, state={})]
-    assert len(records) == 2 + SLICE_DEBUG_LOG_COUNT
+    assert len(records) == 2 + SLICE_DEBUG_LOG_COUNT + TRACE_STATUS_COUNT
     assert [r.record.data for r in records if r.type == Type.RECORD] == [{"value": "23"}, {"value": 23}]
 
 
@@ -520,8 +526,8 @@ def test_read_default_http_availability_strategy_stream_available(catalog, mocke
     source = MockAbstractSource(streams=streams)
     logger = logging.getLogger(f"airbyte.{getattr(abstract_source, 'name', '')}")
     records = [r for r in source.read(logger=logger, config={}, catalog=catalog, state={})]
-    # 3 for http stream and 3 for non http stream
-    assert len(records) == 3 + 3
+    # 3 for http stream, 3 for non http stream and 3 for stream status messages for each stream (2x)
+    assert len(records) == 3 + 3 + 3 + 3
     assert http_stream.read_records.called
     assert non_http_stream.read_records.called
 
@@ -578,8 +584,8 @@ def test_read_default_http_availability_strategy_stream_unavailable(catalog, moc
     with caplog.at_level(logging.WARNING):
         records = [r for r in source.read(logger=logger, config={}, catalog=catalog, state={})]
 
-    # 0 for http stream and 3 for non http stream
-    assert len(records) == 0 + 3
+    # 0 for http stream, 3 for non http stream and 3 status trace meessages
+    assert len(records) == 0 + 3 + 3
     assert non_http_stream.read_records.called
     expected_logs = [
         f"Skipped syncing stream '{http_stream.name}' because it was unavailable.",

unit_tests/utils/test_stream_status_utils.py

@@ -0,0 +1,70 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteStream,
+    AirbyteStreamStatus,
+    ConfiguredAirbyteStream,
+    DestinationSyncMode,
+    SyncMode,
+    TraceType,
+)
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message
+
+stream = AirbyteStream(name="name", namespace="namespace", json_schema={}, supported_sync_modes=[SyncMode.full_refresh])
+configured_stream = ConfiguredAirbyteStream(stream=stream, sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite)
+
+
+def test_started_as_message():
+    stream_status = AirbyteStreamStatus.STARTED
+    airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status)
+
+    assert type(airbyte_message) == AirbyteMessage
+    assert airbyte_message.type == MessageType.TRACE
+    assert airbyte_message.trace.type == TraceType.STREAM_STATUS
+    assert airbyte_message.trace.emitted_at > 0
+    assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name
+    assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace
+    assert airbyte_message.trace.stream_status.status == stream_status
+
+
+def test_running_as_message():
+    stream_status = AirbyteStreamStatus.RUNNING
+    airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status)
+
+    assert type(airbyte_message) == AirbyteMessage
+    assert airbyte_message.type == MessageType.TRACE
+    assert airbyte_message.trace.type == TraceType.STREAM_STATUS
+    assert airbyte_message.trace.emitted_at > 0
+    assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name
+    assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace
+    assert airbyte_message.trace.stream_status.status == stream_status
+
+
+def test_complete_as_message():
+    stream_status = AirbyteStreamStatus.COMPLETE
+    airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status)
+
+    assert type(airbyte_message) == AirbyteMessage
+    assert airbyte_message.type == MessageType.TRACE
+    assert airbyte_message.trace.type == TraceType.STREAM_STATUS
+    assert airbyte_message.trace.emitted_at > 0
+    assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name
+    assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace
+    assert airbyte_message.trace.stream_status.status == stream_status
+
+
+def test_incomplete_failed_as_message():
+    stream_status = AirbyteStreamStatus.INCOMPLETE
+    airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status)
+
+    assert type(airbyte_message) == AirbyteMessage
+    assert airbyte_message.type == MessageType.TRACE
+    assert airbyte_message.trace.type == TraceType.STREAM_STATUS
+    assert airbyte_message.trace.emitted_at > 0
+    assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name
+    assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace
+    assert airbyte_message.trace.stream_status.status == stream_status