airbyte-cdk 0.40.2__py3-none-any.whl → 0.40.4__py3-none-any.whl

airbyte_cdk/config_observation.py

@@ -68,10 +68,18 @@ def observe_connector_config(non_observed_connector_config: MutableMapping[str,
 
 
 def emit_configuration_as_airbyte_control_message(config: MutableMapping):
+    """
+    WARNING: deprecated - emit_configuration_as_airbyte_control_message is being deprecated in favor of the MessageRepository mechanism.
+    See the airbyte_cdk.sources.message package
+    """
+    airbyte_message = create_connector_config_control_message(config)
+    print(airbyte_message.json(exclude_unset=True))
+
+
+def create_connector_config_control_message(config):
     control_message = AirbyteControlMessage(
         type=OrchestratorType.CONNECTOR_CONFIG,
         emitted_at=time.time() * 1000,
         connectorConfig=AirbyteControlConnectorConfigMessage(config=config),
     )
-    airbyte_message = AirbyteMessage(type=Type.CONTROL, control=control_message)
-    print(airbyte_message.json(exclude_unset=True))
+    return AirbyteMessage(type=Type.CONTROL, control=control_message)

airbyte_cdk/connector_builder/message_grouper.py

@@ -14,6 +14,7 @@ from airbyte_cdk.entrypoint import AirbyteEntrypoint
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
 from airbyte_cdk.utils import AirbyteTracedException
+from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer
 from airbyte_cdk.utils.schema_inferrer import SchemaInferrer
 from airbyte_protocol.models.airbyte_protocol import (
     AirbyteControlMessage,
@@ -46,6 +47,7 @@ class MessageGrouper:
         if record_limit is not None and not (1 <= record_limit <= 1000):
             raise ValueError(f"Record limit must be between 1 and 1000. Got {record_limit}")
         schema_inferrer = SchemaInferrer()
+        datetime_format_inferrer = DatetimeFormatInferrer()
 
         if record_limit is None:
             record_limit = self._max_record_limit
@@ -58,6 +60,7 @@ class MessageGrouper:
         for message_group in self._get_message_groups(
                 self._read_stream(source, config, configured_catalog),
                 schema_inferrer,
+                datetime_format_inferrer,
                 record_limit,
         ):
             if isinstance(message_group, AirbyteLogMessage):
@@ -79,11 +82,12 @@ class MessageGrouper:
             inferred_schema=schema_inferrer.get_stream_schema(
                 configured_catalog.streams[0].stream.name
             ),  # The connector builder currently only supports reading from a single stream at a time
-            latest_config_update=latest_config_update.connectorConfig.config if latest_config_update else self._clean_config(config),
+            latest_config_update=self._clean_config(latest_config_update.connectorConfig.config) if latest_config_update else None,
+            inferred_datetime_formats=datetime_format_inferrer.get_inferred_datetime_formats(),
         )
 
     def _get_message_groups(
-            self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, limit: int
+            self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, datetime_format_inferrer: DatetimeFormatInferrer, limit: int
     ) -> Iterable[Union[StreamReadPages, AirbyteControlMessage, AirbyteLogMessage, AirbyteTraceMessage]]:
         """
         Message groups are partitioned according to when request log messages are received. Subsequent response log messages
@@ -141,6 +145,7 @@ class MessageGrouper:
                 current_page_records.append(message.record.data)
                 records_count += 1
                 schema_inferrer.accumulate(message.record)
+                datetime_format_inferrer.accumulate(message.record)
             elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG:
                 yield message.control
         else:

airbyte_cdk/connector_builder/models.py

@@ -48,6 +48,7 @@ class StreamRead(object):
     slices: List[StreamReadSlices]
     test_read_limit_reached: bool
     inferred_schema: Optional[Dict[str, Any]]
+    inferred_datetime_formats: Optional[Dict[str, str]]
     latest_config_update: Optional[Dict[str, Any]]
 
 

airbyte_cdk/entrypoint.py

@@ -77,27 +77,32 @@ class AirbyteEntrypoint(object):
         else:
             self.logger.setLevel(logging.INFO)
 
-        # todo: add try catch for exceptions with different exit codes
         source_spec: ConnectorSpecification = self.source.spec(self.logger)
-        with tempfile.TemporaryDirectory() as temp_dir:
-            if cmd == "spec":
-                message = AirbyteMessage(type=Type.SPEC, spec=source_spec)
-                yield message.json(exclude_unset=True)
-            else:
-                raw_config = self.source.read_config(parsed_args.config)
-                config = self.source.configure(raw_config, temp_dir)
-
-                if cmd == "check":
-                    yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config))
-                elif cmd == "discover":
-                    yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.discover(source_spec, config))
-                elif cmd == "read":
-                    config_catalog = self.source.read_catalog(parsed_args.catalog)
-                    state = self.source.read_state(parsed_args.state)
-
-                    yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.read(source_spec, config, config_catalog, state))
+        try:
+            with tempfile.TemporaryDirectory() as temp_dir:
+                if cmd == "spec":
+                    message = AirbyteMessage(type=Type.SPEC, spec=source_spec)
+                    yield from [
+                        self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source)
+                    ]
+                    yield self.airbyte_message_to_string(message)
                 else:
-                    raise Exception("Unexpected command " + cmd)
+                    raw_config = self.source.read_config(parsed_args.config)
+                    config = self.source.configure(raw_config, temp_dir)
+
+                    if cmd == "check":
+                        yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config))
+                    elif cmd == "discover":
+                        yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.discover(source_spec, config))
+                    elif cmd == "read":
+                        config_catalog = self.source.read_catalog(parsed_args.catalog)
+                        state = self.source.read_state(parsed_args.state)
+
+                        yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.read(source_spec, config, config_catalog, state))
+                    else:
+                        raise Exception("Unexpected command " + cmd)
+        finally:
+            yield from [self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source)]
 
     def check(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterable[AirbyteMessage]:
         self.set_up_secret_filter(config, source_spec.connectionSpecification)
@@ -106,6 +111,7 @@ class AirbyteEntrypoint(object):
         except AirbyteTracedException as traced_exc:
             connection_status = traced_exc.as_connection_status_message()
             if connection_status:
+                yield from self._emit_queued_messages(self.source)
                 yield connection_status
                 return
 
@@ -115,6 +121,7 @@ class AirbyteEntrypoint(object):
         else:
             self.logger.error("Check failed")
 
+        yield from self._emit_queued_messages(self.source)
         yield AirbyteMessage(type=Type.CONNECTION_STATUS, connectionStatus=check_result)
 
     def discover(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterable[AirbyteMessage]:
@@ -122,6 +129,8 @@ class AirbyteEntrypoint(object):
         if self.source.check_config_against_spec:
             self.validate_connection(source_spec, config)
         catalog = self.source.discover(self.logger, config)
+
+        yield from self._emit_queued_messages(self.source)
         yield AirbyteMessage(type=Type.CATALOG, catalog=catalog)
 
     def read(self, source_spec: ConnectorSpecification, config: TConfig, catalog: TCatalog, state: TState) -> Iterable[AirbyteMessage]:
@@ -130,6 +139,7 @@ class AirbyteEntrypoint(object):
             self.validate_connection(source_spec, config)
 
         yield from self.source.read(self.logger, config, catalog, state)
+        yield from self._emit_queued_messages(self.source)
 
     @staticmethod
     def validate_connection(source_spec: ConnectorSpecification, config: Mapping[str, Any]) -> None:
@@ -149,6 +159,11 @@ class AirbyteEntrypoint(object):
     def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> str:
         return airbyte_message.json(exclude_unset=True)
 
+    def _emit_queued_messages(self, source) -> Iterable[AirbyteMessage]:
+        if hasattr(source, "message_repository") and source.message_repository:
+            yield from source.message_repository.consume_queue()
+        return
+
 
 def launch(source: Source, args: List[str]):
     source_entrypoint = AirbyteEntrypoint(source)

airbyte_cdk/sources/abstract_source.py

@@ -22,6 +22,7 @@ from airbyte_cdk.models import (
 )
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
+from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.source import Source
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.core import StreamData
@@ -130,6 +131,7 @@ class AbstractSource(Source, ABC):
                     yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE)
                     raise e
                 except Exception as e:
+                    yield from self._emit_queued_messages()
                     logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}")
                     logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
                     yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE)
@@ -198,6 +200,7 @@ class AbstractSource(Source, ABC):
                     logger.info(f"Marking stream {stream_name} as RUNNING")
                     # If we just read the first record of the stream, emit the transition to the RUNNING state
                     yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.RUNNING)
+            yield from self._emit_queued_messages()
             yield record
 
         logger.info(f"Read {record_counter} records from {stream_name} stream")
@@ -264,6 +267,7 @@ class AbstractSource(Source, ABC):
             record_counter = 0
             for message_counter, record_data_or_message in enumerate(records, start=1):
                 message = self._get_message(record_data_or_message, stream_instance)
+                yield from self._emit_queued_messages()
                 yield message
                 if message.type == MessageType.RECORD:
                     record = message.record
@@ -298,6 +302,11 @@ class AbstractSource(Source, ABC):
         """
         return logger.isEnabledFor(logging.DEBUG)
 
+    def _emit_queued_messages(self):
+        if self.message_repository:
+            yield from self.message_repository.consume_queue()
+        return
+
     def _read_full_refresh(
         self,
         logger: logging.Logger,
@@ -357,3 +366,7 @@ class AbstractSource(Source, ABC):
             return record_data_or_message
         else:
             return stream_data_to_airbyte_message(stream.name, record_data_or_message, stream.transformer, stream.get_json_schema())
+
+    @property
+    def message_repository(self) -> Union[None, MessageRepository]:
+        return None

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -580,10 +580,40 @@ definitions:
           - "{{ config['record_cursor'] }}"
       datetime_format:
         title: Cursor Field Datetime Format
-        description: The datetime format of the Cursor Field.
+        description: |
+          The datetime format of the Cursor Field. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
+            * **%s**: Epoch unix timestamp - `1686218963`
+            * **%a**: Weekday (abbreviated) - `Sun`
+            * **%A**: Weekday (full) - `Sunday`
+            * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)
+            * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`
+            * **%b**: Month (abbreviated) - `Jan`
+            * **%B**: Month (full) - `January`
+            * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`
+            * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`
+            * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`
+            * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`
+            * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`
+            * **%p**: AM/PM indicator
+            * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`
+            * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`
+            * **%f**: Microsecond (zero-padded to 6 digits) - `000000`
+            * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`
+            * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`
+            * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`
+            * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`
+            * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`
+            * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`
+            * **%x**: Date standard format - `08/16/1988`
+            * **%X**: Time standard format - `21:30:00`
+            * **%%**: Literal '%' character
+
+            Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).
         type: string
         examples:
           - "%Y-%m-%dT%H:%M:%S.%f%z"
+          - "%Y-%m-%d"
+          - "%s"
       cursor_granularity:
         title: Cursor Granularity
         description:
@@ -1283,11 +1313,39 @@ definitions:
           - "{{ config['start_time'] }}"
       datetime_format:
         title: Datetime Format
-        description: Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use %s if the datetime value is in epoch time (Unix timestamp).
+        description: |
+          Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
+            * **%s**: Epoch unix timestamp - `1686218963`
+            * **%a**: Weekday (abbreviated) - `Sun`
+            * **%A**: Weekday (full) - `Sunday`
+            * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)
+            * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`
+            * **%b**: Month (abbreviated) - `Jan`
+            * **%B**: Month (full) - `January`
+            * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`
+            * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`
+            * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`
+            * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`
+            * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`
+            * **%p**: AM/PM indicator
+            * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`
+            * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`
+            * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`
+            * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`, `+1030`, `+063415`, `-030712.345216`
+            * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`
+            * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`
+            * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`
+            * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`
+            * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`
+            * **%x**: Date representation - `08/16/1988`
+            * **%X**: Time representation - `21:30:00`
+            * **%%**: Literal '%' character
+
+            Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).
         type: string
         default: ""
         examples:
-          - "%Y-%m-%dT%H:%M:%S.%f%"
+          - "%Y-%m-%dT%H:%M:%S.%f%z"
           - "%Y-%m-%d"
           - "%s"
       max_datetime:

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -26,6 +26,7 @@ from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer impo
 from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver
 from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory
 from airbyte_cdk.sources.declarative.types import ConnectionDefinition
+from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.core import Stream
 from jsonschema.exceptions import ValidationError
 from jsonschema.validators import validate
@@ -61,6 +62,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         self._debug = debug
         self._emit_connector_builder_messages = emit_connector_builder_messages
         self._constructor = component_factory if component_factory else ModelToComponentFactory(emit_connector_builder_messages)
+        self._message_repository = self._constructor.get_message_repository()
 
         self._validate_source()
 
@@ -68,6 +70,10 @@ class ManifestDeclarativeSource(DeclarativeSource):
     def resolved_manifest(self) -> Mapping[str, Any]:
         return self._source_config
 
+    @property
+    def message_repository(self) -> Union[None, MessageRepository]:
+        return self._message_repository
+
     @property
     def connection_checker(self) -> ConnectionChecker:
         check = self._source_config["check"]

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -453,8 +453,8 @@ class MinMaxDatetime(BaseModel):
     )
     datetime_format: Optional[str] = Field(
         "",
-        description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use %s if the datetime value is in epoch time (Unix timestamp).',
-        examples=["%Y-%m-%dT%H:%M:%S.%f%", "%Y-%m-%d", "%s"],
+        description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`, `+1030`, `+063415`, `-030712.345216`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n  * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n  * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date representation - `08/16/1988`\n  * **%X**: Time representation - `21:30:00`\n  * **%%**: Literal \'%\' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
+        examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
         title="Datetime Format",
     )
     max_datetime: Optional[str] = Field(
@@ -806,8 +806,8 @@ class DatetimeBasedCursor(BaseModel):
     )
     datetime_format: str = Field(
         ...,
-        description="The datetime format of the Cursor Field.",
-        examples=["%Y-%m-%dT%H:%M:%S.%f%z"],
+        description="The datetime format of the Cursor Field. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n  * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n  * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date standard format - `08/16/1988`\n  * **%X**: Time standard format - `21:30:00`\n  * **%%**: Literal '%' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
+        examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
         title="Cursor Field Datetime Format",
     )
     cursor_granularity: Optional[str] = Field(

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -100,6 +100,7 @@ from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStrea
 from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields
 from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
 from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.message import InMemoryMessageRepository
 from pydantic import BaseModel
 
 ComponentDefinition: Union[Literal, Mapping, List]
@@ -121,6 +122,7 @@ class ModelToComponentFactory:
         self._limit_slices_fetched = limit_slices_fetched
         self._emit_connector_builder_messages = emit_connector_builder_messages
         self._disable_retries = disable_retries
+        self._message_repository = InMemoryMessageRepository()
 
     def _init_mappings(self):
         self.PYDANTIC_MODEL_TO_CONSTRUCTOR: [Type[BaseModel], Callable] = {
@@ -675,8 +677,7 @@ class ModelToComponentFactory:
     def create_no_pagination(model: NoPaginationModel, config: Config, **kwargs) -> NoPagination:
         return NoPagination(parameters={})
 
-    @staticmethod
-    def create_oauth_authenticator(model: OAuthAuthenticatorModel, config: Config, **kwargs) -> DeclarativeOauth2Authenticator:
+    def create_oauth_authenticator(self, model: OAuthAuthenticatorModel, config: Config, **kwargs) -> DeclarativeOauth2Authenticator:
         if model.refresh_token_updater:
             return DeclarativeSingleUseRefreshTokenOauth2Authenticator(
                 config,
@@ -693,6 +694,7 @@ class ModelToComponentFactory:
                 refresh_request_body=InterpolatedMapping(model.refresh_request_body or {}, parameters=model.parameters).eval(config),
                 scopes=model.scopes,
                 token_expiry_date_format=model.token_expiry_date_format,
+                message_repository=self._message_repository,
             )
         return DeclarativeOauth2Authenticator(
             access_token_name=model.access_token_name,
@@ -845,3 +847,6 @@ class ModelToComponentFactory:
         return WaitUntilTimeFromHeaderBackoffStrategy(
             header=model.header, parameters=model.parameters, config=config, min_wait=model.min_wait, regex=model.regex
         )
+
+    def get_message_repository(self):
+        return self._message_repository

airbyte_cdk/sources/message/__init__.py

@@ -0,0 +1,7 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+from .repository import InMemoryMessageRepository, MessageRepository
+
+__all__ = ["InMemoryMessageRepository", "MessageRepository"]

airbyte_cdk/sources/message/repository.py

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from typing import Iterable
+
+from airbyte_cdk.models import AirbyteMessage, Type
+
+
+class MessageRepository(ABC):
+    @abstractmethod
+    def emit_message(self, message: AirbyteMessage) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def consume_queue(self) -> Iterable[AirbyteMessage]:
+        raise NotImplementedError()
+
+
+class InMemoryMessageRepository(MessageRepository):
+    def __init__(self):
+        self._message_queue = []
+
+    def emit_message(self, message: AirbyteMessage) -> None:
+        """
+        :param message: As of today, only AirbyteControlMessages are supported given that supporting other types of message will need more
+          work and therefore this work has been postponed
+        """
+        if message.type != Type.CONTROL:
+            raise ValueError("As of today, only AirbyteControlMessages are supported as part of the InMemoryMessageRepository")
+        self._message_queue.append(message)
+
+    def consume_queue(self) -> Iterable[AirbyteMessage]:
+        while self._message_queue:
+            yield self._message_queue.pop(0)

airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -79,7 +79,12 @@ class AbstractOauth2Authenticator(AuthBase):
     )
     def _get_refresh_access_token_response(self):
         try:
-            response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), data=self.build_refresh_request_body())
+            response = requests.request(
+                method="POST",
+                url=self.get_token_refresh_endpoint(),
+                data=self.build_refresh_request_body(),
+                headers={"Content-Type": "application/json"},
+            )
             response.raise_for_status()
             return response.json()
         except requests.exceptions.RequestException as e:

airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py

@@ -6,7 +6,8 @@ from typing import Any, List, Mapping, Optional, Sequence, Tuple, Union
 
 import dpath
 import pendulum
-from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message
+from airbyte_cdk.config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message
+from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import AbstractOauth2Authenticator
 
 
@@ -115,6 +116,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"),
         token_expiry_date_config_path: Sequence[str] = ("credentials", "token_expiry_date"),
         token_expiry_date_format: Optional[str] = None,
+        message_repository: MessageRepository = None,
     ):
         """
 
@@ -144,6 +146,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         self._token_expiry_date_format = token_expiry_date_format
         self._refresh_token_name = refresh_token_name
         self._connector_config = connector_config
+        self._message_repository = message_repository
         super().__init__(
             token_refresh_endpoint,
             self.get_client_id(),
@@ -211,7 +214,11 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             self.access_token = new_access_token
             self.set_refresh_token(new_refresh_token)
             self.set_token_expiry_date(new_token_expiry_date)
-            emit_configuration_as_airbyte_control_message(self._connector_config)
+            if self._message_repository:
+                self._message_repository.emit_message(create_connector_config_control_message(self._connector_config))
+            else:
+                # FIXME emit_configuration_as_airbyte_control_message as been deprecated in favor of package airbyte_cdk.sources.message
+                emit_configuration_as_airbyte_control_message(self._connector_config)
         return self.access_token
 
     def refresh_access_token(self) -> Tuple[str, str, str]:

airbyte_cdk/utils/datetime_format_inferrer.py

@@ -0,0 +1,80 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Any, Dict, Union
+
+from airbyte_cdk.models import AirbyteRecordMessage
+from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
+
+
+class DatetimeFormatInferrer:
+    """
+    This class is used to detect toplevel fields in records that might be datetime values, along with the used format.
+    """
+
+    def __init__(self):
+        self._parser = DatetimeParser()
+        self._datetime_candidates: Union[None, Dict[str, str]] = None
+        self._formats = [
+            "%Y-%m-%d",
+            "%Y-%m-%d %H:%M:%S",
+            "%Y-%m-%d %H:%M:%S.%f+00:00",
+            "%Y-%m-%dT%H:%M:%S.%f%z",
+            "%s",
+            "%d/%m/%Y %H:%M",
+            "%Y-%m",
+            "%d-%m-%Y",
+            "%Y-%m-%dT%H:%M:%SZ",
+        ]
+        self._timestamp_heuristic_range = range(1_000_000_000, 2_000_000_000)
+
+    def _can_be_datetime(self, value: Any) -> bool:
+        """Checks if the value can be a datetime. This is the case if the value is a string or an integer between 1_000_000_000 and 2_000_000_000. This is separate from the format check for performance reasons"""
+        if isinstance(value, str) and (not value.isdecimal() or int(value) in self._timestamp_heuristic_range):
+            return True
+        if isinstance(value, int) and value in self._timestamp_heuristic_range:
+            return True
+        return False
+
+    def _matches_format(self, value: Any, format: str) -> bool:
+        """Checks if the value matches the format"""
+        try:
+            self._parser.parse(value, format)
+            return True
+        except ValueError:
+            return False
+
+    def _initialize(self, record: AirbyteRecordMessage):
+        """Initializes the internal state of the class"""
+        self._datetime_candidates = {}
+        for field_name, field_value in record.data.items():
+            if not self._can_be_datetime(field_value):
+                continue
+            for format in self._formats:
+                if self._matches_format(field_value, format):
+                    self._datetime_candidates[field_name] = format
+                    break
+
+    def _validate(self, record: AirbyteRecordMessage):
+        """Validates that the record is consistent with the inferred datetime formats"""
+        for candidate_field_name in list(self._datetime_candidates.keys()):
+            candidate_field_format = self._datetime_candidates[candidate_field_name]
+            current_value = record.data.get(candidate_field_name, None)
+            if (
+                current_value is None
+                or not self._can_be_datetime(current_value)
+                or not self._matches_format(current_value, candidate_field_format)
+            ):
+                self._datetime_candidates.pop(candidate_field_name)
+
+    def accumulate(self, record: AirbyteRecordMessage):
+        """Analyzes the record and updates the internal state of candidate datetime fields"""
+        self._initialize(record) if self._datetime_candidates is None else self._validate(record)
+
+    def get_inferred_datetime_formats(self) -> Dict[str, str]:
+        """
+        Returns the list of candidate datetime fields - the keys are the field names and the values are the inferred datetime formats.
+        For these fields the format was consistent across all visited records.
+        """
+        return self._datetime_candidates or {}

{airbyte_cdk-0.40.2.dist-info → airbyte_cdk-0.40.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.40.2
+Version: 0.40.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.40.2.dist-info → airbyte_cdk-0.40.4.dist-info}/RECORD

@@ -1,32 +1,32 @@
 airbyte_cdk/__init__.py,sha256=OBQWv5rF_QTRpOiP6J8J8oTU-GGrfi18i1PRFpahKks,262
-airbyte_cdk/config_observation.py,sha256=TSA2ulzRCZGmA1AK8hOJGkzayjCHvAIglHwM8vI1uuU,3295
+airbyte_cdk/config_observation.py,sha256=3kjxv8xTwCnub2_fTWnMPRx0E7vly1BUeyXOSK15Ql4,3610
 airbyte_cdk/connector.py,sha256=LtTAmBFV1LBUz_fOEbQ_EvBhyUsz8AGOlDsvK8QOOo0,4396
-airbyte_cdk/entrypoint.py,sha256=cTYsCExsiKAmkhhWQLnmbe-o96mQnK1kriSr5Qm8Ntc,7968
+airbyte_cdk/entrypoint.py,sha256=xQ7jLhElMl-Nl1aWHnlaPbCaVv6UNFuspBUo9w7glbU,8803
 airbyte_cdk/exception_handler.py,sha256=CwkiPdZ1WMOr3CBkvKFyHiyLerXGRqBrVlB4p0OImGI,1125
 airbyte_cdk/logger.py,sha256=4Mi2MEQi1uh59BP9Dxw_UEbZuxaJewqK_jvEU2b10nk,3985
 airbyte_cdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=q8mqQjNqpvHZgwVbNuvSe19o4Aw6MQTuhA2URmdz0K0,5443
 airbyte_cdk/connector_builder/main.py,sha256=jn2gqaYAvd6uDoFe0oVhnY23grm5sL-jfIX6kGvhVxk,2994
-airbyte_cdk/connector_builder/message_grouper.py,sha256=uJGOBhinvbisgAa-bQN3XE2L2xFTeVeykLwDCRYcxgc,12110
-airbyte_cdk/connector_builder/models.py,sha256=yW_j91B-3FYNTNbWjR2ZVYTXBHlskT55uxdAqg7FhAE,1221
+airbyte_cdk/connector_builder/message_grouper.py,sha256=dGU85tsOvHkAoQD2lNHA_ibqdr9MNiGlt60nOCuA6yI,12502
+airbyte_cdk/connector_builder/models.py,sha256=jL2SJIWJTLCbBqobw5Qo8WGS0aN-K9TRmfSpDHM5vYc,1277
 airbyte_cdk/destinations/__init__.py,sha256=0Uxmz3iBAyZJdk_bqUVt2pb0UwRTpFjTnFE6fQFbWKY,126
 airbyte_cdk/destinations/destination.py,sha256=_tIMnKcRQbtIsjVvNOVjfbIxgCNLuBXQwQj8MyVm3BI,5420
 airbyte_cdk/models/__init__.py,sha256=LPQcYdDPwrCXiBPe_jexO4UAcbovIb1V9tHB6I7Un30,633
 airbyte_cdk/models/airbyte_protocol.py,sha256=wKXV_4sCzmUyPndiW7HWAj_A6EDRJyk9cA88xvXGQN0,117
 airbyte_cdk/models/well_known_types.py,sha256=KKfNbow2gdLoC1Z4hcXy_JR8m_acsB2ol7gQuEgjobw,117
 airbyte_cdk/sources/__init__.py,sha256=4j6fLtoRCjcZnojpise4EMmQtV1RepBxoGTBgpz80JA,218
-airbyte_cdk/sources/abstract_source.py,sha256=svXe29SUHKA6WJwKlQlyKkZax2ybaG0wSvIFzaibl24,17262
+airbyte_cdk/sources/abstract_source.py,sha256=IpHvPKhYvv36b-krP9vn1wowrfi9iZdqcxDGbl2-jVE,17743
 airbyte_cdk/sources/config.py,sha256=PYsY7y2u3EUwxLiEb96JnuKwH_E8CuxKggsRO2ZPSRc,856
 airbyte_cdk/sources/connector_state_manager.py,sha256=_R-2QnMGimKL0t5aV4f6P1dgd--TB3abY5Seg1xddXk,10469
 airbyte_cdk/sources/source.py,sha256=N3vHZzdUsBETFsql-YpO-LcgjolT_jcnAuHBhGD6Hqk,4278
 airbyte_cdk/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/create_partial.py,sha256=sUJOwD8hBzW4pxw2XhYlSTMgl-WMc5WpP5Oq_jo3fHw,3371
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=-Kt09XCMs61gEphShtPTMGrqVAamr4cml03_YjDuTLQ,74196
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=O_U5vwYhXP19mkWhjJgRJCTHAPwf6xeOEbNDccUb_wg,78273
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=U2As9PDKmcWDgbsWUo-RetJ9fxQOBlwntWZ0NOgs5Ac,1453
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=0iZSpypxt8bhO3Lmf3BpGRTO7Fp0Q2GI8m8xyJJUjeM,6580
 airbyte_cdk/sources/declarative/exceptions.py,sha256=kTPUA4I2NV4J6HDz-mKPGMrfuc592akJnOyYx38l_QM,176
-airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=vTbRNM8D9P_ChOu1GNvtNRt-PM2L9N5Y0pNRyfVFuZg,9759
+airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=lJCJAHmKPssqnDLAnxU6fuwlNAVm_Ut1EQLTxDy8h1I,10018
 airbyte_cdk/sources/declarative/types.py,sha256=b_RJpL9TyAgxJIRYZx5BxpC39p-WccHKxbAqxWrn9oE,482
 airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256=I9Bs9RDsFT8JNiJWRDjKYhqwvv4pqzgYZtF5hVuTDqI,1684
 airbyte_cdk/sources/declarative/auth/__init__.py,sha256=DyQdO5mdKGsttWdEUqxb6WVgD7zTcvpJz-Oet_VNeBg,201
@@ -60,14 +60,14 @@ airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=dyIM-bzh54
 airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=Dc0F87nElWsz_Ikj938eQ9uqZvyqgFhZ8Dqf_-hvndc,4800
 airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=V6WGKJ9cXX1rjuM4bK3Cs9xEryMlkY2U3FMsSBhrgC8,3098
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=EiYnzwCHZV7EYqMJqcy6xKSeHvTKZBsQndjbEwmiTW4,93
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=7XeAhmGHuNRYK97KwxvbrNXS1Az95O7gOMM3uRlGjrU,50104
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=5RI0o8lTGBt4emHSF-Xsk0FE4LccnMCTBk-LK0PdiMA,53677
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/class_types_registry.py,sha256=bK4a74opm6WHyV7HqOVws6GE5Z7cLNc5MaTha69abIQ,6086
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=y7_G5mM07zxT5YG975kdC2PAja-Uc83pYp8WrV3GNdo,522
 airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py,sha256=W8BcK4KOg4ifNXgsdeIoV4oneHjXBKcPHEZHIC4r-hM,3801
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=H23H3nURCxsvjq66Gn9naffp0HJ1fU03wLFu-5F0AhQ,7701
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=6ukHx0bBrCJm9rek1l_MEfS3U_gdJcM4pJRyifJEOp0,6412
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=LJWolEib5v7IpruG_LHgDYCRID1LbdEEtygMng6EiSw,47818
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=u0dVRPddHEL6OYxFIT3Z6TC-8MiFeKKKw0gPMMpn37Y,48075
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=27sOWhw2LBQs62HchURakHQ2M_mtnOatNgU6q8RUtpU,476
 airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=fa6VtTwSoIkDI3SBoRtVx79opVtJX80_gU9bt31lspc,4785
 airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py,sha256=Fi3ocNZZoYkr0uvRgwoVSqne6enxRvi8DOHrASVK2PQ,1851
@@ -125,6 +125,8 @@ airbyte_cdk/sources/declarative/transformations/transformation.py,sha256=q_FDDDY
 airbyte_cdk/sources/deprecated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/deprecated/base_source.py,sha256=5FafxPLDAh2KNBnKxxlC8QvPRgDYUjmT5OzqEKz8kjI,3524
 airbyte_cdk/sources/deprecated/client.py,sha256=6G2xQZJ2BzMJa-Sq4VdvVM9Dwu11rEEwqHGhmXAb3h4,3560
+airbyte_cdk/sources/message/__init__.py,sha256=WIXPTh8Sx18LhEV6sZ_aI5CDqfjc0b2KPBeKZKwbs6I,193
+airbyte_cdk/sources/message/repository.py,sha256=VMGusWUdxtz6WGs0Lv-ut-CbVR222HdjhHvde1shg3E,1187
 airbyte_cdk/sources/singer/__init__.py,sha256=D3zQSiWT0B9t0kKE4JPZjrcDnP2YnFNJ3dfYqSaxo9w,246
 airbyte_cdk/sources/singer/singer_helpers.py,sha256=q1LmgjFxSnN-dobMy7nikUwcK-9FvW5QQfgTqiclbAE,15649
 airbyte_cdk/sources/singer/source.py,sha256=3YY8UTOXmctvMVUnYmIegmL3_IxF55iGP_bc_s2MZdY,8530
@@ -141,9 +143,9 @@ airbyte_cdk/sources/streams/http/auth/core.py,sha256=_s9wewvvIcOgYjhHGDj_YHApnF5
 airbyte_cdk/sources/streams/http/auth/oauth.py,sha256=zchPWN1utNg02F93f5b4UFI5OXYo8-QhocbsXhLdG4U,4135
 airbyte_cdk/sources/streams/http/auth/token.py,sha256=oU1ul0LsGsPGN_vOJOKw1xX2y_XWULRxjqXu7Rivcr8,1940
 airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
-airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=dw9mmIOf05NDqKzzvRA3tXKjx1LvVGm1tPt8TQhf5Y8,5339
+airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=CRfMunZdowlUyaAgIG76NwUo2xISTjs1AJBbJMaZ-p0,5464
 airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=T0hVF2cBXGgIfrCslvTC1uNm9rNbYjENNl2Cb3mXuSY,961
-airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=Y94eU0Ad8tEnCurW-_vrrAnbbCc0Mo5W38aigr85oEw,11005
+airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=uK5n1oImmFkJJCRukvNNSxCwRcXPV0BAkeOmr5ep6LY,11531
 airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=hDti8DlF_R5YYX95hg9BPogYtG-KUYtOifrFDv_L3Hk,2456
 airbyte_cdk/sources/streams/utils/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/streams/utils/stream_helper.py,sha256=8n1e27DqELN_KRXuWW1IE3ZjE9zvhclNqsKtOosI_Ds,1480
@@ -156,6 +158,7 @@ airbyte_cdk/sources/utils/schema_models.py,sha256=m1vOqNkkVYGblc492wKo11Zm5FK9F0
 airbyte_cdk/sources/utils/transform.py,sha256=4GYmO6bq33HF-a1in0dKQKqUOYI1bWItyuYF875bSQg,9493
 airbyte_cdk/utils/__init__.py,sha256=kFLcs2P-tbPyeVOJS9rOv1jZdnSpjG24ro0CHgt_CIk,215
 airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=q3aDl8T10ufGbeqnUPqbZLxQcHdkf2kDfQK_upWzBbI,2894
+airbyte_cdk/utils/datetime_format_inferrer.py,sha256=1z5lGq_DI9LFrT68ftlJSqndS6i-Rs1PX7T_RBtOJpA,3443
 airbyte_cdk/utils/event_timing.py,sha256=Hn5kCc9xGKLcV5EYpJCZwNiz9neKKu2WG8FJF_hy278,2377
 airbyte_cdk/utils/schema_inferrer.py,sha256=j0us_mEMj8PVVzSZfoS1adK7V7a--mSHQozo6xmsiIc,3720
 airbyte_cdk/utils/stream_status_utils.py,sha256=X1Vy7BhglycjdIWpfKDfwJussNCxYffelKt6Utjx-qY,1005
@@ -163,8 +166,8 @@ airbyte_cdk/utils/traced_exception.py,sha256=9G2sG9eYkvn6Aa7rMuUW_KIRszRaTc_xdnT
 source_declarative_manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 source_declarative_manifest/main.py,sha256=HXzuRsRyhHwPrGU-hc4S7RrgoOoHImqkdfbmO2geBeE,1027
 unit_tests/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-unit_tests/connector_builder/test_connector_builder_handler.py,sha256=V9p7AFECaLqSK-iGvu0OqwV6qREQC2BhWo0H4OoiiK4,26895
-unit_tests/connector_builder/test_message_grouper.py,sha256=XMVRW45RDTgy1YVzkV-jOXj7Ar2mzgDV8OW2QDzZjYU,28510
+unit_tests/connector_builder/test_connector_builder_handler.py,sha256=0TQn6C_De9mpRMU6lcrcuKWIwAHKw2GVMG8iT6OTBMo,28489
+unit_tests/connector_builder/test_message_grouper.py,sha256=MSj9bQd4MtGsmXP-wPHiq4nODbLyrNT-W2CVpNOs2tE,28116
 unit_tests/connector_builder/utils.py,sha256=AAggdGWP-mNuWOZUHLAVIbjTeIcdPo-3pbMm5zdYpS0,796
 unit_tests/destinations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/destinations/test_destination.py,sha256=koG_j812KMkcIxoUH6XlAL3zsephZJmlHvyzJXm0dCs,10269
@@ -172,7 +175,7 @@ unit_tests/singer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 unit_tests/singer/test_singer_helpers.py,sha256=pZV6VxJuK-3-FICNGmoGbokrA_zkaFZEd4rYZCVpSRU,1762
 unit_tests/singer/test_singer_source.py,sha256=edN_kv7dnYAdBveWdUYOs74ak0dK6p8uaX225h_ZILA,4442
 unit_tests/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unit_tests/sources/test_abstract_source.py,sha256=eHZjhfSN-fzqbvdZtGqa5FVwggoFDXxi5SBA1-LQi70,44194
+unit_tests/sources/test_abstract_source.py,sha256=Gn5XJKQlJhxxqawS5T-81BUXmPNUvg3g2UK-kXq-v48,46351
 unit_tests/sources/test_config.py,sha256=gFXqU_6OjwHXkV4JHMqQUznxmvTWN8nAv0w0-FFpugc,2477
 unit_tests/sources/test_connector_state_manager.py,sha256=ynFxA63Cxe6t-wMMh9C6ByTlMAuk8W7H2FikDhnUEQ0,24264
 unit_tests/sources/test_source.py,sha256=eVtU9Zuc9gBsg11Pb5xjDtyU0gVrbYqbZ4RmzPvDw_M,24695
@@ -183,7 +186,7 @@ unit_tests/sources/declarative/test_declarative_stream.py,sha256=3leJnZIYHiFq8XI
 unit_tests/sources/declarative/test_manifest_declarative_source.py,sha256=GckUc3nepzZkD1UM24woHlYCVZb5DP4IAQC3IeMyZF0,58924
 unit_tests/sources/declarative/test_yaml_declarative_source.py,sha256=6HhsUFgB7ueN0yOUHWb4gpPYLng5jasxN_plvz3x37g,5097
 unit_tests/sources/declarative/auth/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-unit_tests/sources/declarative/auth/test_oauth.py,sha256=j-xEUbRPs5jnRAvKCNLKDpEbAZLmXHEy9tSEkYUrYx0,8442
+unit_tests/sources/declarative/auth/test_oauth.py,sha256=WOGs28NVOvb0lIy1ymtQgUEbI8r1Z2fBIY6iWBqCnoE,8514
 unit_tests/sources/declarative/auth/test_session_token_auth.py,sha256=mxWCm_0AyVI6J1Q5CjogXY-EkXFfWkMZjNtBeb0bOow,6135
 unit_tests/sources/declarative/auth/test_token_auth.py,sha256=EIaxGFvaUE6vAUW2_tBrds6nTx4qhfYK8ppRwoNXKd0,6162
 unit_tests/sources/declarative/checks/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
@@ -247,6 +250,8 @@ unit_tests/sources/declarative/schema/source_test/__init__.py,sha256=4Hw-PX1-VgE
 unit_tests/sources/declarative/states/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 unit_tests/sources/declarative/stream_slicers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py,sha256=MI1kLtMuC1LKryBzub0KconsrpIVgPOhAtYM4b3qRfA,9507
+unit_tests/sources/message/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+unit_tests/sources/message/test_repository.py,sha256=qgCFpRUZU_Mm2JePtyIX5KheFYXCDj1ODTlo8z-Yz4Y,2234
 unit_tests/sources/streams/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/streams/test_availability_strategy.py,sha256=vJrSEk9NwRghu0YsSNoMYHKWzA9UFemwyClpke8Mk2s,2315
 unit_tests/sources/streams/test_streams_core.py,sha256=YOC7XqWFJ13Z4YuO9Nh4AR4AwpJ-s111vqPplFfpxk4,5059
@@ -256,14 +261,15 @@ unit_tests/sources/streams/http/test_http.py,sha256=H0lGcb0XHuM1R7GC3wAaaxhGoNwi
 unit_tests/sources/streams/http/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/streams/http/auth/test_auth.py,sha256=gdWpJ-cR64qRXmmPOQWhVd4E6ekXyJEIEfJxA0jlDvc,6546
 unit_tests/sources/streams/http/requests_native_auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py,sha256=_BZVsG_LZUXfBmHWTlKIw65eGkdwFSiKRlpjsccj61U,12396
+unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py,sha256=NoTfDSClXFqjbN_zvoleVWO0lDhjR4obWYn5ApQkWnI,14166
 unit_tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+unit_tests/utils/test_datetime_format_inferrer.py,sha256=Io2o5flTre9gyI_IDDMpzxOjCz3sr16LO0GRqOD59uk,2946
 unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg4MNPAG-xhpk,7817
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
 unit_tests/utils/test_stream_status_utils.py,sha256=NpV155JMXA6CG-2Zvofa14lItobyh3Onttc59X4m5DI,3382
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.40.2.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.40.2.dist-info/METADATA,sha256=Ts5OITHn2vkPFfNNDy5V3dylvBgEZXRPIKVmuCvijZw,8902
-airbyte_cdk-0.40.2.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-airbyte_cdk-0.40.2.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.40.2.dist-info/RECORD,,
+airbyte_cdk-0.40.4.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.40.4.dist-info/METADATA,sha256=cg5ce7pYJVInYpcsugnD0J80AjRXnzzaKY11MEHKBeg,8902
+airbyte_cdk-0.40.4.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+airbyte_cdk-0.40.4.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.40.4.dist-info/RECORD,,

unit_tests/connector_builder/test_connector_builder_handler.py

@@ -176,9 +176,9 @@ def invalid_config_file(tmp_path):
 
 
 def test_handle_resolve_manifest(valid_resolve_manifest_config_file, dummy_catalog):
-    with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patch:
+    with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patched_handle:
         handle_request(["read", "--config", str(valid_resolve_manifest_config_file), "--catalog", str(dummy_catalog)])
-        assert patch.call_count == 1
+        assert patched_handle.call_count == 1
 
 
 def test_handle_test_read(valid_read_config_file, configured_catalog):
@@ -354,6 +354,7 @@ def test_read():
         ],
         test_read_limit_reached=False,
         inferred_schema=None,
+        inferred_datetime_formats=None,
         latest_config_update={}
     )
 
@@ -368,6 +369,7 @@ def test_read():
                 ],
                 "test_read_limit_reached": False,
                 "inferred_schema": None,
+                "inferred_datetime_formats": None,
                 "latest_config_update": {}
             },
             emitted_at=1,
@@ -382,6 +384,37 @@ def test_read():
         assert output_record == expected_airbyte_message
 
 
+def test_config_update():
+    manifest = copy.deepcopy(MANIFEST)
+    manifest["definitions"]["retriever"]["requester"]["authenticator"] = {
+        "type": "OAuthAuthenticator",
+        "token_refresh_endpoint": "https://oauth.endpoint.com/tokens/bearer",
+        "client_id": "{{ config['credentials']['client_id'] }}",
+        "client_secret": "{{ config['credentials']['client_secret'] }}",
+        "refresh_token": "{{ config['credentials']['refresh_token'] }}",
+        "refresh_token_updater": {}
+    }
+    config = copy.deepcopy(TEST_READ_CONFIG)
+    config["__injected_declarative_manifest"] = manifest
+    config["credentials"] = {
+        "client_id": "a client id",
+        "client_secret": "a client secret",
+        "refresh_token": "a refresh token",
+    }
+    source = ManifestDeclarativeSource(manifest)
+
+    refresh_request_response = {
+        "access_token": "an updated access token",
+        "refresh_token": "an updated refresh token",
+        "expires_in": 3600,
+    }
+    with patch("airbyte_cdk.sources.streams.http.requests_native_auth.SingleUseRefreshTokenOauth2Authenticator._get_refresh_access_token_response", return_value=refresh_request_response):
+        output = handle_connector_builder_request(
+            source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), TestReadLimits()
+        )
+        assert output.record.data["latest_config_update"]
+
+
 @patch("traceback.TracebackException.from_exception")
 def test_read_returns_error_response(mock_from_exception):
     class MockManifestDeclarativeSource:
@@ -410,7 +443,8 @@ def test_read_returns_error_response(mock_from_exception):
                                           slice_descriptor=None, state=None)],
                                       test_read_limit_reached=False,
                                       inferred_schema=None,
-                                      latest_config_update={})
+                                      inferred_datetime_formats={},
+                                      latest_config_update=None)
 
     expected_message = AirbyteMessage(
         type=MessageType.RECORD,

unit_tests/connector_builder/test_message_grouper.py

@@ -94,7 +94,8 @@ def test_get_grouped_messages(mock_entrypoint_read):
         "body": {"custom": "field"},
     }
     response = {"status_code": 200, "headers": {"field": "value"}, "body": '{"name": "field"}', "http_method": "GET"}
-    expected_schema = {"$schema": "http://json-schema.org/schema#", "properties": {"name": {"type": "string"}}, "type": "object"}
+    expected_schema = {"$schema": "http://json-schema.org/schema#", "properties": {"name": {"type": "string"}, "date": {"type": "string"}}, "type": "object"}
+    expected_datetime_fields = {"date":"%Y-%m-%d"}
     expected_pages = [
         StreamReadPages(
             request=HttpRequest(
@@ -105,7 +106,7 @@ def test_get_grouped_messages(mock_entrypoint_read):
                 http_method="GET",
             ),
             response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'),
-            records=[{"name": "Shinobu Kocho"}, {"name": "Muichiro Tokito"}],
+            records=[{"name": "Shinobu Kocho", "date": "2023-03-03"}, {"name": "Muichiro Tokito", "date": "2023-03-04"}],
         ),
         StreamReadPages(
             request=HttpRequest(
@@ -116,7 +117,7 @@ def test_get_grouped_messages(mock_entrypoint_read):
                 http_method="GET",
             ),
             response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'),
-            records=[{"name": "Mitsuri Kanroji"}],
+            records=[{"name": "Mitsuri Kanroji", "date": "2023-03-05"}],
         ),
     ]
 
@@ -124,11 +125,11 @@ def test_get_grouped_messages(mock_entrypoint_read):
         [
             request_log_message(request),
             response_log_message(response),
-            record_message("hashiras", {"name": "Shinobu Kocho"}),
-            record_message("hashiras", {"name": "Muichiro Tokito"}),
+            record_message("hashiras", {"name": "Shinobu Kocho", "date": "2023-03-03"}),
+            record_message("hashiras", {"name": "Muichiro Tokito", "date": "2023-03-04"}),
             request_log_message(request),
             response_log_message(response),
-            record_message("hashiras", {"name": "Mitsuri Kanroji"}),
+            record_message("hashiras", {"name": "Mitsuri Kanroji", "date": "2023-03-05"}),
         ]
     ))
 
@@ -138,6 +139,7 @@ def test_get_grouped_messages(mock_entrypoint_read):
     )
 
     assert actual_response.inferred_schema == expected_schema
+    assert actual_response.inferred_datetime_formats == expected_datetime_fields
 
     single_slice = actual_response.slices[0]
     for i, actual_page in enumerate(single_slice.pages):
@@ -552,18 +554,6 @@ def test_given_control_message_then_stream_read_has_config_update(mock_entrypoin
     assert stream_read.latest_config_update == updated_config
 
 
-@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read')
-def test_given_no_control_message_then_use_in_memory_config_change_as_update(mock_entrypoint_read):
-    mock_source = make_mock_source(mock_entrypoint_read, iter(any_request_and_response_with_a_record()))
-    connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
-    full_config = {**CONFIG, **{"__injected_declarative_manifest": MANIFEST}}
-    stream_read: StreamRead = connector_builder_handler.get_message_groups(
-        source=mock_source, config=full_config, configured_catalog=create_configured_catalog("hashiras")
-    )
-
-    assert stream_read.latest_config_update == CONFIG
-
-
 @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read')
 def test_given_multiple_control_messages_then_stream_read_has_latest_based_on_emitted_at(mock_entrypoint_read):
     earliest = 0

unit_tests/sources/declarative/auth/test_oauth.py

@@ -203,7 +203,7 @@ class TestOauth2Authenticator:
             assert oauth.get_token_expiry_date() == pendulum.parse(next_day)
 
 
-def mock_request(method, url, data):
-    if url == "refresh_end":
+def mock_request(method, url, data, headers):
+    if url == "refresh_end" and headers == {"Content-Type": "application/json"}:
         return resp
-    raise Exception(f"Error while refreshing access token with request: {method}, {url}, {data}")
+    raise Exception(f"Error while refreshing access token with request: {method}, {url}, {data}, {headers}")

unit_tests/sources/message/test_repository.py

@@ -0,0 +1,65 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import pytest
+from airbyte_cdk.models import (
+    AirbyteControlConnectorConfigMessage,
+    AirbyteControlMessage,
+    AirbyteLogMessage,
+    AirbyteMessage,
+    Level,
+    OrchestratorType,
+    Type,
+)
+from airbyte_cdk.sources.message import InMemoryMessageRepository
+
+A_CONTROL = AirbyteControlMessage(
+    type=OrchestratorType.CONNECTOR_CONFIG,
+    emitted_at=0,
+    connectorConfig=AirbyteControlConnectorConfigMessage(config={"a config": "value"}),
+)
+ANOTHER_CONTROL = AirbyteControlMessage(
+    type=OrchestratorType.CONNECTOR_CONFIG,
+    emitted_at=0,
+    connectorConfig=AirbyteControlConnectorConfigMessage(config={"another config": "another value"}),
+)
+
+
+def test_given_no_messages_when_consume_queue_then_return_empty():
+    repo = InMemoryMessageRepository()
+    messages = list(repo.consume_queue())
+    assert messages == []
+
+
+def test_given_messages_when_consume_queue_then_return_messages():
+    repo = InMemoryMessageRepository()
+    first_message = AirbyteMessage(type=Type.CONTROL, control=A_CONTROL)
+    repo.emit_message(first_message)
+    second_message = AirbyteMessage(type=Type.CONTROL, control=ANOTHER_CONTROL)
+    repo.emit_message(second_message)
+
+    messages = repo.consume_queue()
+
+    assert list(messages) == [first_message, second_message]
+
+
+def test_given_message_is_consumed_when_consume_queue_then_remove_message_from_queue():
+    repo = InMemoryMessageRepository()
+    first_message = AirbyteMessage(type=Type.CONTROL, control=A_CONTROL)
+    repo.emit_message(first_message)
+    second_message = AirbyteMessage(type=Type.CONTROL, control=ANOTHER_CONTROL)
+    repo.emit_message(second_message)
+
+    message_generator = repo.consume_queue()
+    consumed_message = next(message_generator)
+    assert consumed_message == first_message
+
+    second_message_generator = repo.consume_queue()
+    assert list(second_message_generator) == [second_message]
+
+
+def test_given_message_is_not_control_message_when_emit_message_then_raise_error():
+    repo = InMemoryMessageRepository()
+    with pytest.raises(ValueError):
+        repo.emit_message(AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="any log message")))

unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py

@@ -4,11 +4,13 @@
 
 import json
 import logging
+from unittest.mock import Mock
 
 import freezegun
 import pendulum
 import pytest
 import requests
+from airbyte_cdk.models import OrchestratorType, Type
 from airbyte_cdk.sources.streams.http.requests_native_auth import (
     BasicHttpAuthenticator,
     MultipleTokenAuthenticator,
@@ -243,7 +245,7 @@ class TestSingleUseRefreshTokenOauth2Authenticator:
             ("date_format", "2023-04-04", "YYYY-MM-DD", "2023-04-04T00:00:00+00:00"),
         ]
     )
-    def test_get_access_token(self, test_name, expires_in_value, expiry_date_format, expected_expiry_date, capsys, mocker, connector_config):
+    def test_given_no_message_repository_get_access_token(self, test_name, expires_in_value, expiry_date_format, expected_expiry_date, capsys, mocker, connector_config):
         authenticator = SingleUseRefreshTokenOauth2Authenticator(
             connector_config,
             token_refresh_endpoint="foobar",
@@ -270,6 +272,30 @@ class TestSingleUseRefreshTokenOauth2Authenticator:
         assert not captured.out
         assert authenticator.access_token == access_token == "new_access_token"
 
+    def test_given_message_repository_when_get_access_token_emit_message(self, mocker, connector_config):
+        message_repository = Mock()
+        authenticator = SingleUseRefreshTokenOauth2Authenticator(
+            connector_config,
+            token_refresh_endpoint="foobar",
+            client_id=connector_config["credentials"]["client_id"],
+            client_secret=connector_config["credentials"]["client_secret"],
+            token_expiry_date_format="YYYY-MM-DD",
+            message_repository=message_repository,
+        )
+        authenticator.refresh_access_token = mocker.Mock(return_value=("new_access_token", "2023-04-04", "new_refresh_token"))
+        authenticator.token_has_expired = mocker.Mock(return_value=True)
+
+        authenticator.get_access_token()
+
+        emitted_message = message_repository.emit_message.call_args_list[0].args[0]
+        assert emitted_message.type == Type.CONTROL
+        assert emitted_message.control.type == OrchestratorType.CONNECTOR_CONFIG
+        assert emitted_message.control.connectorConfig.config["credentials"]["access_token"] == "new_access_token"
+        assert emitted_message.control.connectorConfig.config["credentials"]["refresh_token"] == "new_refresh_token"
+        assert emitted_message.control.connectorConfig.config["credentials"]["token_expiry_date"] == "2023-04-04T00:00:00+00:00"
+        assert emitted_message.control.connectorConfig.config["credentials"]["client_id"] == "my_client_id"
+        assert emitted_message.control.connectorConfig.config["credentials"]["client_secret"] == "my_client_secret"
+
     def test_refresh_access_token(self, mocker, connector_config):
         authenticator = SingleUseRefreshTokenOauth2Authenticator(
             connector_config,
@@ -288,7 +314,7 @@ class TestSingleUseRefreshTokenOauth2Authenticator:
         assert authenticator.refresh_access_token() == ("new_access_token", "42", "new_refresh_token")
 
 
-def mock_request(method, url, data):
-    if url == "refresh_end":
+def mock_request(method, url, data, headers):
+    if url == "refresh_end" and headers == {"Content-Type": "application/json"}:
         return resp
-    raise Exception(f"Error while refreshing access token with request: {method}, {url}, {data}")
+    raise Exception(f"Error while refreshing access token with request: {method}, {url}, {data}, {headers}")

unit_tests/sources/test_abstract_source.py

@@ -7,7 +7,7 @@ import datetime
 import logging
 from collections import defaultdict
 from typing import Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union
-from unittest.mock import call
+from unittest.mock import Mock, call
 
 import pytest
 from airbyte_cdk.models import (
@@ -37,9 +37,11 @@ from airbyte_cdk.models import Type
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
+from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams import IncrementalMixin, Stream
 from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+from pytest import fixture
 
 logger = logging.getLogger("airbyte")
 
@@ -50,10 +52,12 @@ class MockSource(AbstractSource):
         check_lambda: Callable[[], Tuple[bool, Optional[Any]]] = None,
         streams: List[Stream] = None,
         per_stream: bool = True,
+        message_repository: MessageRepository = None
     ):
         self._streams = streams
         self.check_lambda = check_lambda
         self.per_stream = per_stream
+        self._message_repository = message_repository
 
     def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
         if self.check_lambda:
@@ -69,6 +73,10 @@ class MockSource(AbstractSource):
     def per_stream_state_enabled(self) -> bool:
         return self.per_stream
 
+    @property
+    def message_repository(self):
+        return self._message_repository
+
 
 class StreamNoStateMethod(Stream):
     name = "managers"
@@ -97,6 +105,16 @@ class MockStreamOverridesStateMethod(Stream, IncrementalMixin):
         self._cursor_value = value.get(self.cursor_field, self.start_date)
 
 
+MESSAGE_FROM_REPOSITORY = Mock()
+
+
+@fixture
+def message_repository():
+    message_repository = Mock(spec=MessageRepository)
+    message_repository.consume_queue.return_value = [message for message in [MESSAGE_FROM_REPOSITORY]]
+    return message_repository
+
+
 def test_successful_check():
     """Tests that if a source returns TRUE for the connection check the appropriate connectionStatus success message is returned"""
     expected = AirbyteConnectionStatus(status=Status.SUCCEEDED)
@@ -221,6 +239,34 @@ def test_read_nonexistent_stream_raises_exception(mocker):
         list(src.read(logger, {}, catalog))
 
 
+def test_read_stream_emits_repository_message_before_record(mocker, message_repository):
+    stream = MockStream(name="my_stream")
+    mocker.patch.object(MockStream, "get_json_schema", return_value={})
+    mocker.patch.object(MockStream, "read_records", side_effect=[[{"a record": "a value"}, {"another record": "another value"}]])
+    message_repository.consume_queue.side_effect = [[message for message in [MESSAGE_FROM_REPOSITORY]], []]
+
+    source = MockSource(streams=[stream], message_repository=message_repository)
+
+    messages = list(source.read(logger, {}, ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)])))
+
+    assert messages.count(MESSAGE_FROM_REPOSITORY) == 1
+    record_messages = (message for message in messages if message.type == Type.RECORD)
+    assert all(messages.index(MESSAGE_FROM_REPOSITORY) < messages.index(record) for record in record_messages)
+
+
+def test_read_stream_emits_repository_message_on_error(mocker, message_repository):
+    stream = MockStream(name="my_stream")
+    mocker.patch.object(MockStream, "get_json_schema", return_value={})
+    mocker.patch.object(MockStream, "read_records", side_effect=RuntimeError("error"))
+    message_repository.consume_queue.return_value = [message for message in [MESSAGE_FROM_REPOSITORY]]
+
+    source = MockSource(streams=[stream], message_repository=message_repository)
+
+    with pytest.raises(RuntimeError):
+        messages = list(source.read(logger, {}, ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)])))
+        assert MESSAGE_FROM_REPOSITORY in messages
+
+
 def test_read_stream_with_error_gets_display_message(mocker):
     stream = MockStream(name="my_stream")
 

unit_tests/utils/test_datetime_format_inferrer.py

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Dict, List
+
+import pytest
+from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage
+from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer
+
+NOW = 1234567
+
+
+@pytest.mark.parametrize(
+    "test_name,input_records,expected_candidate_fields",
+    [
+        ("empty", [], {}),
+        ("simple_match", [{"d": "2022-02-03"}], {"d": "%Y-%m-%d"}),
+        ("timestamp_match_integer", [{"d": 1686058051}], {"d": "%s"}),
+        ("timestamp_match_string", [{"d": "1686058051"}], {"d": "%s"}),
+        ("timestamp_no_match_integer", [{"d": 99}], {}),
+        ("timestamp_no_match_string", [{"d": "99999999999999999999"}], {}),
+        ("simple_no_match", [{"d": "20220203"}], {}),
+        ("multiple_match", [{"d": "2022-02-03", "e": "2022-02-03"}], {"d": "%Y-%m-%d", "e": "%Y-%m-%d"}),
+        (
+            "multiple_no_match",
+            [{"d": "20220203", "r": "ccc", "e": {"something-else": "2023-03-03"}, "s": ["2023-03-03"], "x": False, "y": 123}],
+            {},
+        ),
+        ("format_1", [{"d": "2022-02-03"}], {"d": "%Y-%m-%d"}),
+        ("format_2", [{"d": "2022-02-03 12:34:56"}], {"d": "%Y-%m-%d %H:%M:%S"}),
+        ("format_3", [{"d": "2022-02-03 12:34:56.123456+00:00"}], {"d": "%Y-%m-%d %H:%M:%S.%f+00:00"}),
+        ("format_4", [{"d": "2022-02-03T12:34:56.123456+0000"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}),
+        ("format_4 2", [{"d": "2022-02-03T12:34:56.000Z"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}),
+        ("format_4 2", [{"d": "2022-02-03T12:34:56.000000Z"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}),
+        ("format_6", [{"d": "03/02/2022 12:34"}], {"d": "%d/%m/%Y %H:%M"}),
+        ("format_7", [{"d": "2022-02"}], {"d": "%Y-%m"}),
+        ("format_8", [{"d": "03-02-2022"}], {"d": "%d-%m-%Y"}),
+        ("limit_down", [{"d": "2022-02-03", "x": "2022-02-03"}, {"d": "2022-02-03", "x": "another thing"}], {"d": "%Y-%m-%d"}),
+        ("limit_down all", [{"d": "2022-02-03", "x": "2022-02-03"}, {"d": "also another thing", "x": "another thing"}], {}),
+        ("limit_down empty", [{"d": "2022-02-03", "x": "2022-02-03"}, {}], {}),
+        ("limit_down unsupported type", [{"d": "2022-02-03"}, {"d": False}], {}),
+        ("limit_down complex type", [{"d": "2022-02-03"}, {"d": {"date": "2022-03-03"}}], {}),
+        ("limit_down different format", [{"d": "2022-02-03"}, {"d": 1686058051}], {}),
+        ("limit_down different format", [{"d": "2022-02-03"}, {"d": "2022-02-03T12:34:56.000000Z"}], {}),
+        ("no scope expand", [{}, {"d": "2022-02-03"}], {}),
+    ],
+)
+def test_schema_inferrer(test_name, input_records: List, expected_candidate_fields: Dict[str, str]):
+    inferrer = DatetimeFormatInferrer()
+    for record in input_records:
+        inferrer.accumulate(AirbyteRecordMessage(stream="abc", data=record, emitted_at=NOW))
+    assert inferrer.get_inferred_datetime_formats() == expected_candidate_fields