airbyte-cdk 0.40.2__tar.gz → 0.40.4__tar.gz

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.40.2
+Version: 0.40.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk/config_observation.py

@@ -68,10 +68,18 @@ def observe_connector_config(non_observed_connector_config: MutableMapping[str,
 
 
 def emit_configuration_as_airbyte_control_message(config: MutableMapping):
+    """
+    WARNING: deprecated - emit_configuration_as_airbyte_control_message is being deprecated in favor of the MessageRepository mechanism.
+    See the airbyte_cdk.sources.message package
+    """
+    airbyte_message = create_connector_config_control_message(config)
+    print(airbyte_message.json(exclude_unset=True))
+
+
+def create_connector_config_control_message(config):
     control_message = AirbyteControlMessage(
         type=OrchestratorType.CONNECTOR_CONFIG,
         emitted_at=time.time() * 1000,
         connectorConfig=AirbyteControlConnectorConfigMessage(config=config),
     )
-    airbyte_message = AirbyteMessage(type=Type.CONTROL, control=control_message)
-    print(airbyte_message.json(exclude_unset=True))
+    return AirbyteMessage(type=Type.CONTROL, control=control_message)

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk/connector_builder/message_grouper.py

@@ -14,6 +14,7 @@ from airbyte_cdk.entrypoint import AirbyteEntrypoint
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
 from airbyte_cdk.utils import AirbyteTracedException
+from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer
 from airbyte_cdk.utils.schema_inferrer import SchemaInferrer
 from airbyte_protocol.models.airbyte_protocol import (
     AirbyteControlMessage,
@@ -46,6 +47,7 @@ class MessageGrouper:
         if record_limit is not None and not (1 <= record_limit <= 1000):
             raise ValueError(f"Record limit must be between 1 and 1000. Got {record_limit}")
         schema_inferrer = SchemaInferrer()
+        datetime_format_inferrer = DatetimeFormatInferrer()
 
         if record_limit is None:
             record_limit = self._max_record_limit
@@ -58,6 +60,7 @@ class MessageGrouper:
         for message_group in self._get_message_groups(
                 self._read_stream(source, config, configured_catalog),
                 schema_inferrer,
+                datetime_format_inferrer,
                 record_limit,
         ):
             if isinstance(message_group, AirbyteLogMessage):
@@ -79,11 +82,12 @@ class MessageGrouper:
             inferred_schema=schema_inferrer.get_stream_schema(
                 configured_catalog.streams[0].stream.name
             ),  # The connector builder currently only supports reading from a single stream at a time
-            latest_config_update=latest_config_update.connectorConfig.config if latest_config_update else self._clean_config(config),
+            latest_config_update=self._clean_config(latest_config_update.connectorConfig.config) if latest_config_update else None,
+            inferred_datetime_formats=datetime_format_inferrer.get_inferred_datetime_formats(),
         )
 
     def _get_message_groups(
-            self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, limit: int
+            self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, datetime_format_inferrer: DatetimeFormatInferrer, limit: int
     ) -> Iterable[Union[StreamReadPages, AirbyteControlMessage, AirbyteLogMessage, AirbyteTraceMessage]]:
         """
         Message groups are partitioned according to when request log messages are received. Subsequent response log messages
@@ -141,6 +145,7 @@ class MessageGrouper:
                 current_page_records.append(message.record.data)
                 records_count += 1
                 schema_inferrer.accumulate(message.record)
+                datetime_format_inferrer.accumulate(message.record)
             elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG:
                 yield message.control
         else:

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk/connector_builder/models.py

@@ -48,6 +48,7 @@ class StreamRead(object):
     slices: List[StreamReadSlices]
     test_read_limit_reached: bool
     inferred_schema: Optional[Dict[str, Any]]
+    inferred_datetime_formats: Optional[Dict[str, str]]
     latest_config_update: Optional[Dict[str, Any]]
 
 

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk/entrypoint.py

@@ -77,27 +77,32 @@ class AirbyteEntrypoint(object):
         else:
             self.logger.setLevel(logging.INFO)
 
-        # todo: add try catch for exceptions with different exit codes
         source_spec: ConnectorSpecification = self.source.spec(self.logger)
-        with tempfile.TemporaryDirectory() as temp_dir:
-            if cmd == "spec":
-                message = AirbyteMessage(type=Type.SPEC, spec=source_spec)
-                yield message.json(exclude_unset=True)
-            else:
-                raw_config = self.source.read_config(parsed_args.config)
-                config = self.source.configure(raw_config, temp_dir)
-
-                if cmd == "check":
-                    yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config))
-                elif cmd == "discover":
-                    yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.discover(source_spec, config))
-                elif cmd == "read":
-                    config_catalog = self.source.read_catalog(parsed_args.catalog)
-                    state = self.source.read_state(parsed_args.state)
-
-                    yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.read(source_spec, config, config_catalog, state))
+        try:
+            with tempfile.TemporaryDirectory() as temp_dir:
+                if cmd == "spec":
+                    message = AirbyteMessage(type=Type.SPEC, spec=source_spec)
+                    yield from [
+                        self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source)
+                    ]
+                    yield self.airbyte_message_to_string(message)
                 else:
-                    raise Exception("Unexpected command " + cmd)
+                    raw_config = self.source.read_config(parsed_args.config)
+                    config = self.source.configure(raw_config, temp_dir)
+
+                    if cmd == "check":
+                        yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config))
+                    elif cmd == "discover":
+                        yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.discover(source_spec, config))
+                    elif cmd == "read":
+                        config_catalog = self.source.read_catalog(parsed_args.catalog)
+                        state = self.source.read_state(parsed_args.state)
+
+                        yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.read(source_spec, config, config_catalog, state))
+                    else:
+                        raise Exception("Unexpected command " + cmd)
+        finally:
+            yield from [self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source)]
 
     def check(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterable[AirbyteMessage]:
         self.set_up_secret_filter(config, source_spec.connectionSpecification)
@@ -106,6 +111,7 @@ class AirbyteEntrypoint(object):
         except AirbyteTracedException as traced_exc:
             connection_status = traced_exc.as_connection_status_message()
             if connection_status:
+                yield from self._emit_queued_messages(self.source)
                 yield connection_status
                 return
 
@@ -115,6 +121,7 @@ class AirbyteEntrypoint(object):
         else:
             self.logger.error("Check failed")
 
+        yield from self._emit_queued_messages(self.source)
         yield AirbyteMessage(type=Type.CONNECTION_STATUS, connectionStatus=check_result)
 
     def discover(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterable[AirbyteMessage]:
@@ -122,6 +129,8 @@ class AirbyteEntrypoint(object):
         if self.source.check_config_against_spec:
             self.validate_connection(source_spec, config)
         catalog = self.source.discover(self.logger, config)
+
+        yield from self._emit_queued_messages(self.source)
         yield AirbyteMessage(type=Type.CATALOG, catalog=catalog)
 
     def read(self, source_spec: ConnectorSpecification, config: TConfig, catalog: TCatalog, state: TState) -> Iterable[AirbyteMessage]:
@@ -130,6 +139,7 @@ class AirbyteEntrypoint(object):
             self.validate_connection(source_spec, config)
 
         yield from self.source.read(self.logger, config, catalog, state)
+        yield from self._emit_queued_messages(self.source)
 
     @staticmethod
     def validate_connection(source_spec: ConnectorSpecification, config: Mapping[str, Any]) -> None:
@@ -149,6 +159,11 @@ class AirbyteEntrypoint(object):
     def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> str:
         return airbyte_message.json(exclude_unset=True)
 
+    def _emit_queued_messages(self, source) -> Iterable[AirbyteMessage]:
+        if hasattr(source, "message_repository") and source.message_repository:
+            yield from source.message_repository.consume_queue()
+        return
+
 
 def launch(source: Source, args: List[str]):
     source_entrypoint = AirbyteEntrypoint(source)

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/abstract_source.py

@@ -22,6 +22,7 @@ from airbyte_cdk.models import (
 )
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
+from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.source import Source
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.core import StreamData
@@ -130,6 +131,7 @@ class AbstractSource(Source, ABC):
                     yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE)
                     raise e
                 except Exception as e:
+                    yield from self._emit_queued_messages()
                     logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}")
                     logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED")
                     yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE)
@@ -198,6 +200,7 @@ class AbstractSource(Source, ABC):
                     logger.info(f"Marking stream {stream_name} as RUNNING")
                     # If we just read the first record of the stream, emit the transition to the RUNNING state
                     yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.RUNNING)
+            yield from self._emit_queued_messages()
             yield record
 
         logger.info(f"Read {record_counter} records from {stream_name} stream")
@@ -264,6 +267,7 @@ class AbstractSource(Source, ABC):
             record_counter = 0
             for message_counter, record_data_or_message in enumerate(records, start=1):
                 message = self._get_message(record_data_or_message, stream_instance)
+                yield from self._emit_queued_messages()
                 yield message
                 if message.type == MessageType.RECORD:
                     record = message.record
@@ -298,6 +302,11 @@ class AbstractSource(Source, ABC):
         """
         return logger.isEnabledFor(logging.DEBUG)
 
+    def _emit_queued_messages(self):
+        if self.message_repository:
+            yield from self.message_repository.consume_queue()
+        return
+
     def _read_full_refresh(
         self,
         logger: logging.Logger,
@@ -357,3 +366,7 @@ class AbstractSource(Source, ABC):
             return record_data_or_message
         else:
             return stream_data_to_airbyte_message(stream.name, record_data_or_message, stream.transformer, stream.get_json_schema())
+
+    @property
+    def message_repository(self) -> Union[None, MessageRepository]:
+        return None

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -580,10 +580,40 @@ definitions:
           - "{{ config['record_cursor'] }}"
       datetime_format:
         title: Cursor Field Datetime Format
-        description: The datetime format of the Cursor Field.
+        description: |
+          The datetime format of the Cursor Field. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
+            * **%s**: Epoch unix timestamp - `1686218963`
+            * **%a**: Weekday (abbreviated) - `Sun`
+            * **%A**: Weekday (full) - `Sunday`
+            * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)
+            * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`
+            * **%b**: Month (abbreviated) - `Jan`
+            * **%B**: Month (full) - `January`
+            * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`
+            * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`
+            * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`
+            * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`
+            * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`
+            * **%p**: AM/PM indicator
+            * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`
+            * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`
+            * **%f**: Microsecond (zero-padded to 6 digits) - `000000`
+            * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`
+            * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`
+            * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`
+            * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`
+            * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`
+            * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`
+            * **%x**: Date standard format - `08/16/1988`
+            * **%X**: Time standard format - `21:30:00`
+            * **%%**: Literal '%' character
+
+            Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).
         type: string
         examples:
           - "%Y-%m-%dT%H:%M:%S.%f%z"
+          - "%Y-%m-%d"
+          - "%s"
       cursor_granularity:
         title: Cursor Granularity
         description:
@@ -1283,11 +1313,39 @@ definitions:
           - "{{ config['start_time'] }}"
       datetime_format:
         title: Datetime Format
-        description: Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use %s if the datetime value is in epoch time (Unix timestamp).
+        description: |
+          Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
+            * **%s**: Epoch unix timestamp - `1686218963`
+            * **%a**: Weekday (abbreviated) - `Sun`
+            * **%A**: Weekday (full) - `Sunday`
+            * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)
+            * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`
+            * **%b**: Month (abbreviated) - `Jan`
+            * **%B**: Month (full) - `January`
+            * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`
+            * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`
+            * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`
+            * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`
+            * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`
+            * **%p**: AM/PM indicator
+            * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`
+            * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`
+            * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`
+            * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`, `+1030`, `+063415`, `-030712.345216`
+            * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`
+            * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`
+            * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`
+            * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`
+            * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`
+            * **%x**: Date representation - `08/16/1988`
+            * **%X**: Time representation - `21:30:00`
+            * **%%**: Literal '%' character
+
+            Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).
         type: string
         default: ""
         examples:
-          - "%Y-%m-%dT%H:%M:%S.%f%"
+          - "%Y-%m-%dT%H:%M:%S.%f%z"
           - "%Y-%m-%d"
           - "%s"
       max_datetime:

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -26,6 +26,7 @@ from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer impo
 from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver
 from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory
 from airbyte_cdk.sources.declarative.types import ConnectionDefinition
+from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.core import Stream
 from jsonschema.exceptions import ValidationError
 from jsonschema.validators import validate
@@ -61,6 +62,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         self._debug = debug
         self._emit_connector_builder_messages = emit_connector_builder_messages
         self._constructor = component_factory if component_factory else ModelToComponentFactory(emit_connector_builder_messages)
+        self._message_repository = self._constructor.get_message_repository()
 
         self._validate_source()
 
@@ -68,6 +70,10 @@ class ManifestDeclarativeSource(DeclarativeSource):
     def resolved_manifest(self) -> Mapping[str, Any]:
         return self._source_config
 
+    @property
+    def message_repository(self) -> Union[None, MessageRepository]:
+        return self._message_repository
+
     @property
     def connection_checker(self) -> ConnectionChecker:
         check = self._source_config["check"]

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -453,8 +453,8 @@ class MinMaxDatetime(BaseModel):
     )
     datetime_format: Optional[str] = Field(
         "",
-        description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use %s if the datetime value is in epoch time (Unix timestamp).',
-        examples=["%Y-%m-%dT%H:%M:%S.%f%", "%Y-%m-%d", "%s"],
+        description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`, `+1030`, `+063415`, `-030712.345216`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n  * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n  * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date representation - `08/16/1988`\n  * **%X**: Time representation - `21:30:00`\n  * **%%**: Literal \'%\' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
+        examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
         title="Datetime Format",
     )
     max_datetime: Optional[str] = Field(
@@ -806,8 +806,8 @@ class DatetimeBasedCursor(BaseModel):
     )
     datetime_format: str = Field(
         ...,
-        description="The datetime format of the Cursor Field.",
-        examples=["%Y-%m-%dT%H:%M:%S.%f%z"],
+        description="The datetime format of the Cursor Field. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n  * **%s**: Epoch unix timestamp - `1686218963`\n  * **%a**: Weekday (abbreviated) - `Sun`\n  * **%A**: Weekday (full) - `Sunday`\n  * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n  * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n  * **%b**: Month (abbreviated) - `Jan`\n  * **%B**: Month (full) - `January`\n  * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n  * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n  * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n  * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n  * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n  * **%p**: AM/PM indicator\n  * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n  * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n  * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n  * **%z**: UTC offset - `(empty)`, `+0000`, `-0400`\n  * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n  * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n  * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n  * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n  * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n  * **%x**: Date standard format - `08/16/1988`\n  * **%X**: Time standard format - `21:30:00`\n  * **%%**: Literal '%' character\n\n  Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
+        examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
         title="Cursor Field Datetime Format",
     )
     cursor_granularity: Optional[str] = Field(

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -100,6 +100,7 @@ from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStrea
 from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields
 from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
 from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.message import InMemoryMessageRepository
 from pydantic import BaseModel
 
 ComponentDefinition: Union[Literal, Mapping, List]
@@ -121,6 +122,7 @@ class ModelToComponentFactory:
         self._limit_slices_fetched = limit_slices_fetched
         self._emit_connector_builder_messages = emit_connector_builder_messages
         self._disable_retries = disable_retries
+        self._message_repository = InMemoryMessageRepository()
 
     def _init_mappings(self):
         self.PYDANTIC_MODEL_TO_CONSTRUCTOR: [Type[BaseModel], Callable] = {
@@ -675,8 +677,7 @@ class ModelToComponentFactory:
     def create_no_pagination(model: NoPaginationModel, config: Config, **kwargs) -> NoPagination:
         return NoPagination(parameters={})
 
-    @staticmethod
-    def create_oauth_authenticator(model: OAuthAuthenticatorModel, config: Config, **kwargs) -> DeclarativeOauth2Authenticator:
+    def create_oauth_authenticator(self, model: OAuthAuthenticatorModel, config: Config, **kwargs) -> DeclarativeOauth2Authenticator:
         if model.refresh_token_updater:
             return DeclarativeSingleUseRefreshTokenOauth2Authenticator(
                 config,
@@ -693,6 +694,7 @@ class ModelToComponentFactory:
                 refresh_request_body=InterpolatedMapping(model.refresh_request_body or {}, parameters=model.parameters).eval(config),
                 scopes=model.scopes,
                 token_expiry_date_format=model.token_expiry_date_format,
+                message_repository=self._message_repository,
             )
         return DeclarativeOauth2Authenticator(
             access_token_name=model.access_token_name,
@@ -845,3 +847,6 @@ class ModelToComponentFactory:
         return WaitUntilTimeFromHeaderBackoffStrategy(
             header=model.header, parameters=model.parameters, config=config, min_wait=model.min_wait, regex=model.regex
         )
+
+    def get_message_repository(self):
+        return self._message_repository

airbyte-cdk-0.40.4/airbyte_cdk/sources/message/__init__.py

@@ -0,0 +1,7 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+from .repository import InMemoryMessageRepository, MessageRepository
+
+__all__ = ["InMemoryMessageRepository", "MessageRepository"]

airbyte-cdk-0.40.4/airbyte_cdk/sources/message/repository.py

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from typing import Iterable
+
+from airbyte_cdk.models import AirbyteMessage, Type
+
+
+class MessageRepository(ABC):
+    @abstractmethod
+    def emit_message(self, message: AirbyteMessage) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def consume_queue(self) -> Iterable[AirbyteMessage]:
+        raise NotImplementedError()
+
+
+class InMemoryMessageRepository(MessageRepository):
+    def __init__(self):
+        self._message_queue = []
+
+    def emit_message(self, message: AirbyteMessage) -> None:
+        """
+        :param message: As of today, only AirbyteControlMessages are supported given that supporting other types of message will need more
+          work and therefore this work has been postponed
+        """
+        if message.type != Type.CONTROL:
+            raise ValueError("As of today, only AirbyteControlMessages are supported as part of the InMemoryMessageRepository")
+        self._message_queue.append(message)
+
+    def consume_queue(self) -> Iterable[AirbyteMessage]:
+        while self._message_queue:
+            yield self._message_queue.pop(0)

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -79,7 +79,12 @@ class AbstractOauth2Authenticator(AuthBase):
     )
     def _get_refresh_access_token_response(self):
         try:
-            response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), data=self.build_refresh_request_body())
+            response = requests.request(
+                method="POST",
+                url=self.get_token_refresh_endpoint(),
+                data=self.build_refresh_request_body(),
+                headers={"Content-Type": "application/json"},
+            )
             response.raise_for_status()
             return response.json()
         except requests.exceptions.RequestException as e:

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py

@@ -6,7 +6,8 @@ from typing import Any, List, Mapping, Optional, Sequence, Tuple, Union
 
 import dpath
 import pendulum
-from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message
+from airbyte_cdk.config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message
+from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import AbstractOauth2Authenticator
 
 
@@ -115,6 +116,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"),
         token_expiry_date_config_path: Sequence[str] = ("credentials", "token_expiry_date"),
         token_expiry_date_format: Optional[str] = None,
+        message_repository: MessageRepository = None,
     ):
         """
 
@@ -144,6 +146,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         self._token_expiry_date_format = token_expiry_date_format
         self._refresh_token_name = refresh_token_name
         self._connector_config = connector_config
+        self._message_repository = message_repository
         super().__init__(
             token_refresh_endpoint,
             self.get_client_id(),
@@ -211,7 +214,11 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             self.access_token = new_access_token
             self.set_refresh_token(new_refresh_token)
             self.set_token_expiry_date(new_token_expiry_date)
-            emit_configuration_as_airbyte_control_message(self._connector_config)
+            if self._message_repository:
+                self._message_repository.emit_message(create_connector_config_control_message(self._connector_config))
+            else:
+                # FIXME emit_configuration_as_airbyte_control_message as been deprecated in favor of package airbyte_cdk.sources.message
+                emit_configuration_as_airbyte_control_message(self._connector_config)
         return self.access_token
 
     def refresh_access_token(self) -> Tuple[str, str, str]:

airbyte-cdk-0.40.4/airbyte_cdk/utils/datetime_format_inferrer.py

@@ -0,0 +1,80 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Any, Dict, Union
+
+from airbyte_cdk.models import AirbyteRecordMessage
+from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
+
+
+class DatetimeFormatInferrer:
+    """
+    This class is used to detect toplevel fields in records that might be datetime values, along with the used format.
+    """
+
+    def __init__(self):
+        self._parser = DatetimeParser()
+        self._datetime_candidates: Union[None, Dict[str, str]] = None
+        self._formats = [
+            "%Y-%m-%d",
+            "%Y-%m-%d %H:%M:%S",
+            "%Y-%m-%d %H:%M:%S.%f+00:00",
+            "%Y-%m-%dT%H:%M:%S.%f%z",
+            "%s",
+            "%d/%m/%Y %H:%M",
+            "%Y-%m",
+            "%d-%m-%Y",
+            "%Y-%m-%dT%H:%M:%SZ",
+        ]
+        self._timestamp_heuristic_range = range(1_000_000_000, 2_000_000_000)
+
+    def _can_be_datetime(self, value: Any) -> bool:
+        """Checks if the value can be a datetime. This is the case if the value is a string or an integer between 1_000_000_000 and 2_000_000_000. This is separate from the format check for performance reasons"""
+        if isinstance(value, str) and (not value.isdecimal() or int(value) in self._timestamp_heuristic_range):
+            return True
+        if isinstance(value, int) and value in self._timestamp_heuristic_range:
+            return True
+        return False
+
+    def _matches_format(self, value: Any, format: str) -> bool:
+        """Checks if the value matches the format"""
+        try:
+            self._parser.parse(value, format)
+            return True
+        except ValueError:
+            return False
+
+    def _initialize(self, record: AirbyteRecordMessage):
+        """Initializes the internal state of the class"""
+        self._datetime_candidates = {}
+        for field_name, field_value in record.data.items():
+            if not self._can_be_datetime(field_value):
+                continue
+            for format in self._formats:
+                if self._matches_format(field_value, format):
+                    self._datetime_candidates[field_name] = format
+                    break
+
+    def _validate(self, record: AirbyteRecordMessage):
+        """Validates that the record is consistent with the inferred datetime formats"""
+        for candidate_field_name in list(self._datetime_candidates.keys()):
+            candidate_field_format = self._datetime_candidates[candidate_field_name]
+            current_value = record.data.get(candidate_field_name, None)
+            if (
+                current_value is None
+                or not self._can_be_datetime(current_value)
+                or not self._matches_format(current_value, candidate_field_format)
+            ):
+                self._datetime_candidates.pop(candidate_field_name)
+
+    def accumulate(self, record: AirbyteRecordMessage):
+        """Analyzes the record and updates the internal state of candidate datetime fields"""
+        self._initialize(record) if self._datetime_candidates is None else self._validate(record)
+
+    def get_inferred_datetime_formats(self) -> Dict[str, str]:
+        """
+        Returns the list of candidate datetime fields - the keys are the field names and the values are the inferred datetime formats.
+        For these fields the format was consistent across all visited records.
+        """
+        return self._datetime_candidates or {}

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.40.2
+Version: 0.40.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/airbyte_cdk.egg-info/SOURCES.txt

@@ -134,6 +134,8 @@ airbyte_cdk/sources/declarative/transformations/transformation.py
 airbyte_cdk/sources/deprecated/__init__.py
 airbyte_cdk/sources/deprecated/base_source.py
 airbyte_cdk/sources/deprecated/client.py
+airbyte_cdk/sources/message/__init__.py
+airbyte_cdk/sources/message/repository.py
 airbyte_cdk/sources/singer/__init__.py
 airbyte_cdk/sources/singer/singer_helpers.py
 airbyte_cdk/sources/singer/source.py
@@ -165,6 +167,7 @@ airbyte_cdk/sources/utils/schema_models.py
 airbyte_cdk/sources/utils/transform.py
 airbyte_cdk/utils/__init__.py
 airbyte_cdk/utils/airbyte_secrets_utils.py
+airbyte_cdk/utils/datetime_format_inferrer.py
 airbyte_cdk/utils/event_timing.py
 airbyte_cdk/utils/schema_inferrer.py
 airbyte_cdk/utils/stream_status_utils.py
@@ -256,6 +259,8 @@ unit_tests/sources/declarative/schema/source_test/__init__.py
 unit_tests/sources/declarative/states/__init__.py
 unit_tests/sources/declarative/stream_slicers/__init__.py
 unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py
+unit_tests/sources/message/__init__.py
+unit_tests/sources/message/test_repository.py
 unit_tests/sources/streams/__init__.py
 unit_tests/sources/streams/test_availability_strategy.py
 unit_tests/sources/streams/test_streams_core.py
@@ -267,6 +272,7 @@ unit_tests/sources/streams/http/auth/test_auth.py
 unit_tests/sources/streams/http/requests_native_auth/__init__.py
 unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py
 unit_tests/utils/__init__.py
+unit_tests/utils/test_datetime_format_inferrer.py
 unit_tests/utils/test_schema_inferrer.py
 unit_tests/utils/test_secret_utils.py
 unit_tests/utils/test_stream_status_utils.py

{airbyte-cdk-0.40.2 → airbyte-cdk-0.40.4}/setup.py

@@ -17,7 +17,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.40.2",
+    version="0.40.4",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",