airbyte-cdk 0.40.3__tar.gz → 0.40.4__tar.gz

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.40.3
+Version: 0.40.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk/config_observation.py

@@ -68,10 +68,18 @@ def observe_connector_config(non_observed_connector_config: MutableMapping[str,
 
 
 def emit_configuration_as_airbyte_control_message(config: MutableMapping):
+    """
+    WARNING: deprecated - emit_configuration_as_airbyte_control_message is being deprecated in favor of the MessageRepository mechanism.
+    See the airbyte_cdk.sources.message package
+    """
+    airbyte_message = create_connector_config_control_message(config)
+    print(airbyte_message.json(exclude_unset=True))
+
+
+def create_connector_config_control_message(config):
     control_message = AirbyteControlMessage(
         type=OrchestratorType.CONNECTOR_CONFIG,
         emitted_at=time.time() * 1000,
         connectorConfig=AirbyteControlConnectorConfigMessage(config=config),
     )
-    airbyte_message = AirbyteMessage(type=Type.CONTROL, control=control_message)
-    print(airbyte_message.json(exclude_unset=True))
+    return AirbyteMessage(type=Type.CONTROL, control=control_message)

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk/connector_builder/message_grouper.py

@@ -82,7 +82,7 @@ class MessageGrouper:
             inferred_schema=schema_inferrer.get_stream_schema(
                 configured_catalog.streams[0].stream.name
             ),  # The connector builder currently only supports reading from a single stream at a time
-            latest_config_update=latest_config_update.connectorConfig.config if latest_config_update else self._clean_config(config),
+            latest_config_update=self._clean_config(latest_config_update.connectorConfig.config) if latest_config_update else None,
             inferred_datetime_formats=datetime_format_inferrer.get_inferred_datetime_formats(),
         )
 

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk/entrypoint.py

@@ -77,27 +77,32 @@ class AirbyteEntrypoint(object):
         else:
             self.logger.setLevel(logging.INFO)
 
-        # todo: add try catch for exceptions with different exit codes
         source_spec: ConnectorSpecification = self.source.spec(self.logger)
-        with tempfile.TemporaryDirectory() as temp_dir:
-            if cmd == "spec":
-                message = AirbyteMessage(type=Type.SPEC, spec=source_spec)
-                yield message.json(exclude_unset=True)
-            else:
-                raw_config = self.source.read_config(parsed_args.config)
-                config = self.source.configure(raw_config, temp_dir)
-
-                if cmd == "check":
-                    yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config))
-                elif cmd == "discover":
-                    yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.discover(source_spec, config))
-                elif cmd == "read":
-                    config_catalog = self.source.read_catalog(parsed_args.catalog)
-                    state = self.source.read_state(parsed_args.state)
-
-                    yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.read(source_spec, config, config_catalog, state))
+        try:
+            with tempfile.TemporaryDirectory() as temp_dir:
+                if cmd == "spec":
+                    message = AirbyteMessage(type=Type.SPEC, spec=source_spec)
+                    yield from [
+                        self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source)
+                    ]
+                    yield self.airbyte_message_to_string(message)
                 else:
-                    raise Exception("Unexpected command " + cmd)
+                    raw_config = self.source.read_config(parsed_args.config)
+                    config = self.source.configure(raw_config, temp_dir)
+
+                    if cmd == "check":
+                        yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config))
+                    elif cmd == "discover":
+                        yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.discover(source_spec, config))
+                    elif cmd == "read":
+                        config_catalog = self.source.read_catalog(parsed_args.catalog)
+                        state = self.source.read_state(parsed_args.state)
+
+                        yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.read(source_spec, config, config_catalog, state))
+                    else:
+                        raise Exception("Unexpected command " + cmd)
+        finally:
+            yield from [self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source)]
 
     def check(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterable[AirbyteMessage]:
         self.set_up_secret_filter(config, source_spec.connectionSpecification)
@@ -106,6 +111,7 @@ class AirbyteEntrypoint(object):
         except AirbyteTracedException as traced_exc:
             connection_status = traced_exc.as_connection_status_message()
             if connection_status:
+                yield from self._emit_queued_messages(self.source)
                 yield connection_status
                 return
 
@@ -115,6 +121,7 @@ class AirbyteEntrypoint(object):
         else:
             self.logger.error("Check failed")
 
+        yield from self._emit_queued_messages(self.source)
         yield AirbyteMessage(type=Type.CONNECTION_STATUS, connectionStatus=check_result)
 
     def discover(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterable[AirbyteMessage]:
@@ -122,6 +129,8 @@ class AirbyteEntrypoint(object):
         if self.source.check_config_against_spec:
             self.validate_connection(source_spec, config)
         catalog = self.source.discover(self.logger, config)
+
+        yield from self._emit_queued_messages(self.source)
         yield AirbyteMessage(type=Type.CATALOG, catalog=catalog)
 
     def read(self, source_spec: ConnectorSpecification, config: TConfig, catalog: TCatalog, state: TState) -> Iterable[AirbyteMessage]:
@@ -130,6 +139,7 @@ class AirbyteEntrypoint(object):
             self.validate_connection(source_spec, config)
 
         yield from self.source.read(self.logger, config, catalog, state)
+        yield from self._emit_queued_messages(self.source)
 
     @staticmethod
     def validate_connection(source_spec: ConnectorSpecification, config: Mapping[str, Any]) -> None:
@@ -149,6 +159,11 @@ class AirbyteEntrypoint(object):
     def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> str:
         return airbyte_message.json(exclude_unset=True)
 
+    def _emit_queued_messages(self, source) -> Iterable[AirbyteMessage]:
+        if hasattr(source, "message_repository") and source.message_repository:
+            yield from source.message_repository.consume_queue()
+        return
+
 
 def launch(source: Source, args: List[str]):
     source_entrypoint = AirbyteEntrypoint(source)

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/abstract_source.py

@@ -22,6 +22,7 @@ from airbyte_cdk.models import (
 )
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
+from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.source import Source
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.core import StreamData
@@ -130,6 +131,7 @@ class AbstractSource(Source, ABC):
                     yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE)
                     raise e
                 except Exception as e:
+                    yield from self._emit_queued_messages()
                     logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}")
                     logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
                     yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE)
@@ -198,6 +200,7 @@ class AbstractSource(Source, ABC):
                     logger.info(f"Marking stream {stream_name} as RUNNING")
                     # If we just read the first record of the stream, emit the transition to the RUNNING state
                     yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.RUNNING)
+            yield from self._emit_queued_messages()
             yield record
 
         logger.info(f"Read {record_counter} records from {stream_name} stream")
@@ -264,6 +267,7 @@ class AbstractSource(Source, ABC):
             record_counter = 0
             for message_counter, record_data_or_message in enumerate(records, start=1):
                 message = self._get_message(record_data_or_message, stream_instance)
+                yield from self._emit_queued_messages()
                 yield message
                 if message.type == MessageType.RECORD:
                     record = message.record
@@ -298,6 +302,11 @@ class AbstractSource(Source, ABC):
         """
         return logger.isEnabledFor(logging.DEBUG)
 
+    def _emit_queued_messages(self):
+        if self.message_repository:
+            yield from self.message_repository.consume_queue()
+        return
+
     def _read_full_refresh(
         self,
         logger: logging.Logger,
@@ -357,3 +366,7 @@ class AbstractSource(Source, ABC):
             return record_data_or_message
         else:
             return stream_data_to_airbyte_message(stream.name, record_data_or_message, stream.transformer, stream.get_json_schema())
+
+    @property
+    def message_repository(self) -> Union[None, MessageRepository]:
+        return None

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -580,10 +580,40 @@ definitions:
           - "{{ config['record_cursor'] }}"
       datetime_format:
         title: Cursor Field Datetime Format
-        description: The datetime format of the Cursor Field.
+        description: |
+          The datetime format of the Cursor Field. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
+            * **%s**: Epoch unix timestamp - `1686218963`
+            * **%a**: Weekday (abbreviated) - `Sun`
+            * **%A**: Weekday (full) - `Sunday`
+            * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)
+            * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`
+            * **%b**: Month (abbreviated) - `Jan`
+            * **%B**: Month (full) - `January`
+            * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`
+            * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`
+            * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`
+            * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`
+            * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`
+            * **%p**: AM/PM indicator
+            * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`
+            * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`
+            * **%f**: Microsecond (zero-padded to 6 digits) - `000000`
+            * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`
+            * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`
+            * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`
+            * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`
+            * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`
+            * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`
+            * **%x**: Date standard format - `08/16/1988`
+            * **%X**: Time standard format - `21:30:00`
+            * **%%**: Literal '%' character
+
+            Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).
         type: string
         examples:
           - "%Y-%m-%dT%H:%M:%S.%f%z"
+          - "%Y-%m-%d"
+          - "%s"
       cursor_granularity:
         title: Cursor Granularity
         description:
@@ -1283,11 +1313,39 @@ definitions:
           - "{{ config['start_time'] }}"
       datetime_format:
         title: Datetime Format
-        description: Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use %s if the datetime value is in epoch time (Unix timestamp).
+        description: |
+          Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
+            * **%s**: Epoch unix timestamp - `1686218963`
+            * **%a**: Weekday (abbreviated) - `Sun`
+            * **%A**: Weekday (full) - `Sunday`
+            * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)
+            * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`
+            * **%b**: Month (abbreviated) - `Jan`
+            * **%B**: Month (full) - `January`
+            * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`
+            * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`
+            * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`
+            * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`
+            * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`
+            * **%p**: AM/PM indicator
+            * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`
+            * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`
+            * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`
+            * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`, `+1030`, `+063415`, `-030712.345216`
+            * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`
+            * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`
+            * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`
+            * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`
+            * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`
+            * **%x**: Date representation - `08/16/1988`
+            * **%X**: Time representation - `21:30:00`
+            * **%%**: Literal '%' character
+
+            Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).
         type: string
         default: ""
         examples:
-          - "%Y-%m-%dT%H:%M:%S.%f%"
+          - "%Y-%m-%dT%H:%M:%S.%f%z"
           - "%Y-%m-%d"
           - "%s"
       max_datetime:

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -26,6 +26,7 @@ from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer impo
 from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver
 from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory
 from airbyte_cdk.sources.declarative.types import ConnectionDefinition
+from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.core import Stream
 from jsonschema.exceptions import ValidationError
 from jsonschema.validators import validate
@@ -61,6 +62,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         self._debug = debug
         self._emit_connector_builder_messages = emit_connector_builder_messages
         self._constructor = component_factory if component_factory else ModelToComponentFactory(emit_connector_builder_messages)
+        self._message_repository = self._constructor.get_message_repository()
 
         self._validate_source()
 
@@ -68,6 +70,10 @@ class ManifestDeclarativeSource(DeclarativeSource):
     def resolved_manifest(self) -> Mapping[str, Any]:
         return self._source_config
 
+    @property
+    def message_repository(self) -> Union[None, MessageRepository]:
+        return self._message_repository
+
     @property
     def connection_checker(self) -> ConnectionChecker:
         check = self._source_config["check"]

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -453,8 +453,8 @@ class MinMaxDatetime(BaseModel):
     )
     datetime_format: Optional[str] = Field(
         "",
-        description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use %s if the datetime value is in epoch time (Unix timestamp).',
-        examples=["%Y-%m-%dT%H:%M:%S.%f%", "%Y-%m-%d", "%s"],
+        description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`, `+1030`, `+063415`, `-030712.345216`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n  * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n  * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date representation - `08/16/1988`\n  * **%X**: Time representation - `21:30:00`\n  * **%%**: Literal \'%\' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
+        examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
         title="Datetime Format",
     )
     max_datetime: Optional[str] = Field(
@@ -806,8 +806,8 @@ class DatetimeBasedCursor(BaseModel):
     )
     datetime_format: str = Field(
         ...,
-        description="The datetime format of the Cursor Field.",
-        examples=["%Y-%m-%dT%H:%M:%S.%f%z"],
+        description="The datetime format of the Cursor Field. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n  * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n  * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date standard format - `08/16/1988`\n  * **%X**: Time standard format - `21:30:00`\n  * **%%**: Literal '%' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
+        examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
         title="Cursor Field Datetime Format",
     )
     cursor_granularity: Optional[str] = Field(

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -100,6 +100,7 @@ from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStrea
 from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields
 from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
 from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.message import InMemoryMessageRepository
 from pydantic import BaseModel
 
 ComponentDefinition: Union[Literal, Mapping, List]
@@ -121,6 +122,7 @@ class ModelToComponentFactory:
         self._limit_slices_fetched = limit_slices_fetched
         self._emit_connector_builder_messages = emit_connector_builder_messages
         self._disable_retries = disable_retries
+        self._message_repository = InMemoryMessageRepository()
 
     def _init_mappings(self):
         self.PYDANTIC_MODEL_TO_CONSTRUCTOR: [Type[BaseModel], Callable] = {
@@ -675,8 +677,7 @@ class ModelToComponentFactory:
     def create_no_pagination(model: NoPaginationModel, config: Config, **kwargs) -> NoPagination:
         return NoPagination(parameters={})
 
-    @staticmethod
-    def create_oauth_authenticator(model: OAuthAuthenticatorModel, config: Config, **kwargs) -> DeclarativeOauth2Authenticator:
+    def create_oauth_authenticator(self, model: OAuthAuthenticatorModel, config: Config, **kwargs) -> DeclarativeOauth2Authenticator:
         if model.refresh_token_updater:
             return DeclarativeSingleUseRefreshTokenOauth2Authenticator(
                 config,
@@ -693,6 +694,7 @@ class ModelToComponentFactory:
                 refresh_request_body=InterpolatedMapping(model.refresh_request_body or {}, parameters=model.parameters).eval(config),
                 scopes=model.scopes,
                 token_expiry_date_format=model.token_expiry_date_format,
+                message_repository=self._message_repository,
             )
         return DeclarativeOauth2Authenticator(
             access_token_name=model.access_token_name,
@@ -845,3 +847,6 @@ class ModelToComponentFactory:
         return WaitUntilTimeFromHeaderBackoffStrategy(
             header=model.header, parameters=model.parameters, config=config, min_wait=model.min_wait, regex=model.regex
         )
+
+    def get_message_repository(self):
+        return self._message_repository

airbyte-cdk-0.40.4/airbyte_cdk/sources/message/__init__.py

@@ -0,0 +1,7 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+from .repository import InMemoryMessageRepository, MessageRepository
+
+__all__ = ["InMemoryMessageRepository", "MessageRepository"]

airbyte-cdk-0.40.4/airbyte_cdk/sources/message/repository.py

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from typing import Iterable
+
+from airbyte_cdk.models import AirbyteMessage, Type
+
+
+class MessageRepository(ABC):
+    @abstractmethod
+    def emit_message(self, message: AirbyteMessage) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def consume_queue(self) -> Iterable[AirbyteMessage]:
+        raise NotImplementedError()
+
+
+class InMemoryMessageRepository(MessageRepository):
+    def __init__(self):
+        self._message_queue = []
+
+    def emit_message(self, message: AirbyteMessage) -> None:
+        """
+        :param message: As of today, only AirbyteControlMessages are supported given that supporting other types of message will need more
+          work and therefore this work has been postponed
+        """
+        if message.type != Type.CONTROL:
+            raise ValueError("As of today, only AirbyteControlMessages are supported as part of the InMemoryMessageRepository")
+        self._message_queue.append(message)
+
+    def consume_queue(self) -> Iterable[AirbyteMessage]:
+        while self._message_queue:
+            yield self._message_queue.pop(0)

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -79,7 +79,12 @@ class AbstractOauth2Authenticator(AuthBase):
     )
     def _get_refresh_access_token_response(self):
         try:
-            response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), data=self.build_refresh_request_body())
+            response = requests.request(
+                method="POST",
+                url=self.get_token_refresh_endpoint(),
+                data=self.build_refresh_request_body(),
+                headers={"Content-Type": "application/json"},
+            )
             response.raise_for_status()
             return response.json()
         except requests.exceptions.RequestException as e:

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py

@@ -6,7 +6,8 @@ from typing import Any, List, Mapping, Optional, Sequence, Tuple, Union
 
 import dpath
 import pendulum
-from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message
+from airbyte_cdk.config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message
+from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import AbstractOauth2Authenticator
 
 
@@ -115,6 +116,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"),
         token_expiry_date_config_path: Sequence[str] = ("credentials", "token_expiry_date"),
         token_expiry_date_format: Optional[str] = None,
+        message_repository: MessageRepository = None,
     ):
         """
 
@@ -144,6 +146,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         self._token_expiry_date_format = token_expiry_date_format
         self._refresh_token_name = refresh_token_name
         self._connector_config = connector_config
+        self._message_repository = message_repository
         super().__init__(
             token_refresh_endpoint,
             self.get_client_id(),
@@ -211,7 +214,11 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             self.access_token = new_access_token
             self.set_refresh_token(new_refresh_token)
             self.set_token_expiry_date(new_token_expiry_date)
-            emit_configuration_as_airbyte_control_message(self._connector_config)
+            if self._message_repository:
+                self._message_repository.emit_message(create_connector_config_control_message(self._connector_config))
+            else:
+                # FIXME emit_configuration_as_airbyte_control_message as been deprecated in favor of package airbyte_cdk.sources.message
+                emit_configuration_as_airbyte_control_message(self._connector_config)
         return self.access_token
 
     def refresh_access_token(self) -> Tuple[str, str, str]:

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.40.3
+Version: 0.40.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/airbyte_cdk.egg-info/SOURCES.txt

@@ -134,6 +134,8 @@ airbyte_cdk/sources/declarative/transformations/transformation.py
 airbyte_cdk/sources/deprecated/__init__.py
 airbyte_cdk/sources/deprecated/base_source.py
 airbyte_cdk/sources/deprecated/client.py
+airbyte_cdk/sources/message/__init__.py
+airbyte_cdk/sources/message/repository.py
 airbyte_cdk/sources/singer/__init__.py
 airbyte_cdk/sources/singer/singer_helpers.py
 airbyte_cdk/sources/singer/source.py
@@ -257,6 +259,8 @@ unit_tests/sources/declarative/schema/source_test/__init__.py
 unit_tests/sources/declarative/states/__init__.py
 unit_tests/sources/declarative/stream_slicers/__init__.py
 unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py
+unit_tests/sources/message/__init__.py
+unit_tests/sources/message/test_repository.py
 unit_tests/sources/streams/__init__.py
 unit_tests/sources/streams/test_availability_strategy.py
 unit_tests/sources/streams/test_streams_core.py

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/setup.py

@@ -17,7 +17,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.40.3",
+    version="0.40.4",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/unit_tests/connector_builder/test_connector_builder_handler.py

@@ -176,9 +176,9 @@ def invalid_config_file(tmp_path):
 
 
 def test_handle_resolve_manifest(valid_resolve_manifest_config_file, dummy_catalog):
-    with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patch:
+    with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patched_handle:
         handle_request(["read", "--config", str(valid_resolve_manifest_config_file), "--catalog", str(dummy_catalog)])
-        assert patch.call_count == 1
+        assert patched_handle.call_count == 1
 
 
 def test_handle_test_read(valid_read_config_file, configured_catalog):
@@ -384,6 +384,37 @@ def test_read():
         assert output_record == expected_airbyte_message
 
 
+def test_config_update():
+    manifest = copy.deepcopy(MANIFEST)
+    manifest["definitions"]["retriever"]["requester"]["authenticator"] = {
+        "type": "OAuthAuthenticator",
+        "token_refresh_endpoint": "https://oauth.endpoint.com/tokens/bearer",
+        "client_id": "{{ config['credentials']['client_id'] }}",
+        "client_secret": "{{ config['credentials']['client_secret'] }}",
+        "refresh_token": "{{ config['credentials']['refresh_token'] }}",
+        "refresh_token_updater": {}
+    }
+    config = copy.deepcopy(TEST_READ_CONFIG)
+    config["__injected_declarative_manifest"] = manifest
+    config["credentials"] = {
+        "client_id": "a client id",
+        "client_secret": "a client secret",
+        "refresh_token": "a refresh token",
+    }
+    source = ManifestDeclarativeSource(manifest)
+
+    refresh_request_response = {
+        "access_token": "an updated access token",
+        "refresh_token": "an updated refresh token",
+        "expires_in": 3600,
+    }
+    with patch("airbyte_cdk.sources.streams.http.requests_native_auth.SingleUseRefreshTokenOauth2Authenticator._get_refresh_access_token_response", return_value=refresh_request_response):
+        output = handle_connector_builder_request(
+            source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), TestReadLimits()
+        )
+        assert output.record.data["latest_config_update"]
+
+
 @patch("traceback.TracebackException.from_exception")
 def test_read_returns_error_response(mock_from_exception):
     class MockManifestDeclarativeSource:
@@ -413,7 +444,7 @@ def test_read_returns_error_response(mock_from_exception):
                                       test_read_limit_reached=False,
                                       inferred_schema=None,
                                       inferred_datetime_formats={},
-                                      latest_config_update={})
+                                      latest_config_update=None)
 
     expected_message = AirbyteMessage(
         type=MessageType.RECORD,

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/unit_tests/connector_builder/test_message_grouper.py

@@ -554,18 +554,6 @@ def test_given_control_message_then_stream_read_has_config_update(mock_entrypoin
     assert stream_read.latest_config_update == updated_config
 
 
-@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read')
-def test_given_no_control_message_then_use_in_memory_config_change_as_update(mock_entrypoint_read):
-    mock_source = make_mock_source(mock_entrypoint_read, iter(any_request_and_response_with_a_record()))
-    connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
-    full_config = {**CONFIG, **{"__injected_declarative_manifest": MANIFEST}}
-    stream_read: StreamRead = connector_builder_handler.get_message_groups(
-        source=mock_source, config=full_config, configured_catalog=create_configured_catalog("hashiras")
-    )
-
-    assert stream_read.latest_config_update == CONFIG
-
-
 @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read')
 def test_given_multiple_control_messages_then_stream_read_has_latest_based_on_emitted_at(mock_entrypoint_read):
     earliest = 0

{airbyte-cdk-0.40.3 → airbyte-cdk-0.40.4}/unit_tests/sources/declarative/auth/test_oauth.py

@@ -203,7 +203,7 @@ class TestOauth2Authenticator:
             assert oauth.get_token_expiry_date() == pendulum.parse(next_day)
 
 
-def mock_request(method, url, data):
-    if url == "refresh_end":
+def mock_request(method, url, data, headers):
+    if url == "refresh_end" and headers == {"Content-Type": "application/json"}:
         return resp
-    raise Exception(f"Error while refreshing access token with request: {method}, {url}, {data}")
+    raise Exception(f"Error while refreshing access token with request: {method}, {url}, {data}, {headers}")