airbyte-cdk 0.39.3__py3-none-any.whl → 0.39.4__py3-none-any.whl

airbyte_cdk/connector_builder/message_grouper.py

@@ -16,11 +16,13 @@ from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
 from airbyte_cdk.utils import AirbyteTracedException
 from airbyte_cdk.utils.schema_inferrer import SchemaInferrer
 from airbyte_protocol.models.airbyte_protocol import (
+    AirbyteControlMessage,
     AirbyteLogMessage,
     AirbyteMessage,
     AirbyteTraceMessage,
     ConfiguredAirbyteCatalog,
     Level,
+    OrchestratorType,
     TraceType,
 )
 from airbyte_protocol.models.airbyte_protocol import Type as MessageType
@@ -52,6 +54,7 @@ class MessageGrouper:
 
         slices = []
         log_messages = []
+        latest_config_update: AirbyteControlMessage = None
         for message_group in self._get_message_groups(
                 self._read_stream(source, config, configured_catalog),
                 schema_inferrer,
@@ -63,7 +66,9 @@ class MessageGrouper:
                 if message_group.type == TraceType.ERROR:
                     error_message = f"{message_group.error.message} - {message_group.error.stack_trace}"
                     log_messages.append(LogMessage(**{"message": error_message, "level": "ERROR"}))
-
+            elif isinstance(message_group, AirbyteControlMessage):
+                if not latest_config_update or latest_config_update.emitted_at <= message_group.emitted_at:
+                    latest_config_update = message_group
             else:
                 slices.append(message_group)
 
@@ -74,11 +79,12 @@ class MessageGrouper:
             inferred_schema=schema_inferrer.get_stream_schema(
                 configured_catalog.streams[0].stream.name
             ),  # The connector builder currently only supports reading from a single stream at a time
+            latest_config_update=latest_config_update.connectorConfig.config if latest_config_update else self._clean_config(config),
         )
 
     def _get_message_groups(
             self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, limit: int
-    ) -> Iterable[Union[StreamReadPages, AirbyteLogMessage, AirbyteTraceMessage]]:
+    ) -> Iterable[Union[StreamReadPages, AirbyteControlMessage, AirbyteLogMessage, AirbyteTraceMessage]]:
         """
         Message groups are partitioned according to when request log messages are received. Subsequent response log messages
         and record messages belong to the prior request log message and when we encounter another request, append the latest
@@ -135,6 +141,8 @@ class MessageGrouper:
                 current_page_records.append(message.record.data)
                 records_count += 1
                 schema_inferrer.accumulate(message.record)
+            elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG:
+                yield message.control
         else:
             self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records, validate_page_complete=not had_error)
             yield StreamReadSlices(pages=current_slice_pages, slice_descriptor=current_slice_descriptor)
@@ -217,20 +225,10 @@ class MessageGrouper:
     def _parse_slice_description(self, log_message):
         return json.loads(log_message.replace(AbstractSource.SLICE_LOG_PREFIX, "", 1))
 
-    @classmethod
-    def _create_configure_catalog(cls, stream_name: str) -> ConfiguredAirbyteCatalog:
-        return ConfiguredAirbyteCatalog.parse_obj(
-            {
-                "streams": [
-                    {
-                        "stream": {
-                            "name": stream_name,
-                            "json_schema": {},
-                            "supported_sync_modes": ["full_refresh", "incremental"],
-                        },
-                        "sync_mode": "full_refresh",
-                        "destination_sync_mode": "overwrite",
-                    }
-                ]
-            }
-        )
+    @staticmethod
+    def _clean_config(config: Mapping[str, Any]):
+        cleaned_config = deepcopy(config)
+        for key in config.keys():
+            if key.startswith("__"):
+                del cleaned_config[key]
+        return cleaned_config

airbyte_cdk/connector_builder/models.py

@@ -48,6 +48,7 @@ class StreamRead(object):
     slices: List[StreamReadSlices]
     test_read_limit_reached: bool
     inferred_schema: Optional[Dict[str, Any]]
+    latest_config_update: Optional[Dict[str, Any]]
 
 
 @dataclass

{airbyte_cdk-0.39.3.dist-info → airbyte_cdk-0.39.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.39.3
+Version: 0.39.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.39.3.dist-info → airbyte_cdk-0.39.4.dist-info}/RECORD

@@ -8,8 +8,8 @@ airbyte_cdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=q8mqQjNqpvHZgwVbNuvSe19o4Aw6MQTuhA2URmdz0K0,5443
 airbyte_cdk/connector_builder/main.py,sha256=jn2gqaYAvd6uDoFe0oVhnY23grm5sL-jfIX6kGvhVxk,2994
-airbyte_cdk/connector_builder/message_grouper.py,sha256=BcQxrTmeRrF6phR2NcfapgjyPWYRBWGDI5cWuLZSsPg,11851
-airbyte_cdk/connector_builder/models.py,sha256=pdS057gg7N8kp-7LP5BIyC2chNktUhAzpnJLNksuQ3w,1170
+airbyte_cdk/connector_builder/message_grouper.py,sha256=uJGOBhinvbisgAa-bQN3XE2L2xFTeVeykLwDCRYcxgc,12110
+airbyte_cdk/connector_builder/models.py,sha256=yW_j91B-3FYNTNbWjR2ZVYTXBHlskT55uxdAqg7FhAE,1221
 airbyte_cdk/destinations/__init__.py,sha256=0Uxmz3iBAyZJdk_bqUVt2pb0UwRTpFjTnFE6fQFbWKY,126
 airbyte_cdk/destinations/destination.py,sha256=_tIMnKcRQbtIsjVvNOVjfbIxgCNLuBXQwQj8MyVm3BI,5420
 airbyte_cdk/models/__init__.py,sha256=LPQcYdDPwrCXiBPe_jexO4UAcbovIb1V9tHB6I7Un30,633
@@ -163,8 +163,8 @@ airbyte_cdk/utils/traced_exception.py,sha256=9G2sG9eYkvn6Aa7rMuUW_KIRszRaTc_xdnT
 source_declarative_manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 source_declarative_manifest/main.py,sha256=HXzuRsRyhHwPrGU-hc4S7RrgoOoHImqkdfbmO2geBeE,1027
 unit_tests/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-unit_tests/connector_builder/test_connector_builder_handler.py,sha256=iTUz7Iw_eT2yPEwzaoEQxWx92irgsaSuTATsNVw6ym8,26757
-unit_tests/connector_builder/test_message_grouper.py,sha256=VNbvRn3U3Ltsh9TyXFQ74Aw4dN1XbNpSWsx4FsW1r3c,24361
+unit_tests/connector_builder/test_connector_builder_handler.py,sha256=V9p7AFECaLqSK-iGvu0OqwV6qREQC2BhWo0H4OoiiK4,26895
+unit_tests/connector_builder/test_message_grouper.py,sha256=XMVRW45RDTgy1YVzkV-jOXj7Ar2mzgDV8OW2QDzZjYU,28510
 unit_tests/connector_builder/utils.py,sha256=AAggdGWP-mNuWOZUHLAVIbjTeIcdPo-3pbMm5zdYpS0,796
 unit_tests/destinations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/destinations/test_destination.py,sha256=koG_j812KMkcIxoUH6XlAL3zsephZJmlHvyzJXm0dCs,10269
@@ -262,8 +262,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
 unit_tests/utils/test_stream_status_utils.py,sha256=NpV155JMXA6CG-2Zvofa14lItobyh3Onttc59X4m5DI,3382
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.39.3.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.39.3.dist-info/METADATA,sha256=E39UiRsHG-2J4xFqFNRTAz8AnLGFC_3NKuybsieM7Rg,8902
-airbyte_cdk-0.39.3.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-airbyte_cdk-0.39.3.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.39.3.dist-info/RECORD,,
+airbyte_cdk-0.39.4.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.39.4.dist-info/METADATA,sha256=pIdFzow6PMTMXmX-gel8XxP1dX6FwtRScbO35opWUzM,8902
+airbyte_cdk-0.39.4.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+airbyte_cdk-0.39.4.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.39.4.dist-info/RECORD,,

unit_tests/connector_builder/test_connector_builder_handler.py

@@ -354,6 +354,7 @@ def test_read():
         ],
         test_read_limit_reached=False,
         inferred_schema=None,
+        latest_config_update={}
     )
 
     expected_airbyte_message = AirbyteMessage(
@@ -367,6 +368,7 @@ def test_read():
                 ],
                 "test_read_limit_reached": False,
                 "inferred_schema": None,
+                "latest_config_update": {}
             },
             emitted_at=1,
         ),
@@ -407,7 +409,8 @@ def test_read_returns_error_response(mock_from_exception):
                                           pages=[StreamReadPages(records=[], request=None, response=None)],
                                           slice_descriptor=None, state=None)],
                                       test_read_limit_reached=False,
-                                      inferred_schema=None)
+                                      inferred_schema=None,
+                                      latest_config_update={})
 
     expected_message = AirbyteMessage(
         type=MessageType.RECORD,

unit_tests/connector_builder/test_message_grouper.py

@@ -9,7 +9,15 @@ from unittest.mock import MagicMock, patch
 import pytest
 from airbyte_cdk.connector_builder.message_grouper import MessageGrouper
 from airbyte_cdk.connector_builder.models import HttpRequest, HttpResponse, LogMessage, StreamRead, StreamReadPages
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteRecordMessage, Level
+from airbyte_cdk.models import (
+    AirbyteControlConnectorConfigMessage,
+    AirbyteControlMessage,
+    AirbyteLogMessage,
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    Level,
+    OrchestratorType,
+)
 from airbyte_cdk.models import Type as MessageType
 from unit_tests.connector_builder.utils import create_configured_catalog
 
@@ -463,9 +471,9 @@ def test_get_grouped_messages_with_many_slices(mock_entrypoint_read):
         )
     )
 
-    connecto_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
+    connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
 
-    stream_read: StreamRead = connecto_builder_handler.get_message_groups(
+    stream_read: StreamRead = connector_builder_handler.get_message_groups(
         source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")
     )
 
@@ -530,6 +538,76 @@ def test_read_stream_returns_error_if_stream_does_not_exist():
     assert "ERROR" in actual_response.logs[0].level
 
 
+@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read')
+def test_given_control_message_then_stream_read_has_config_update(mock_entrypoint_read):
+    updated_config = {"x": 1}
+    mock_source = make_mock_source(mock_entrypoint_read, iter(
+        any_request_and_response_with_a_record() + [connector_configuration_control_message(1, updated_config)]
+    ))
+    connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
+    stream_read: StreamRead = connector_builder_handler.get_message_groups(
+        source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")
+    )
+
+    assert stream_read.latest_config_update == updated_config
+
+
+@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read')
+def test_given_no_control_message_then_use_in_memory_config_change_as_update(mock_entrypoint_read):
+    mock_source = make_mock_source(mock_entrypoint_read, iter(any_request_and_response_with_a_record()))
+    connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
+    full_config = {**CONFIG, **{"__injected_declarative_manifest": MANIFEST}}
+    stream_read: StreamRead = connector_builder_handler.get_message_groups(
+        source=mock_source, config=full_config, configured_catalog=create_configured_catalog("hashiras")
+    )
+
+    assert stream_read.latest_config_update == CONFIG
+
+
+@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read')
+def test_given_multiple_control_messages_then_stream_read_has_latest_based_on_emitted_at(mock_entrypoint_read):
+    earliest = 0
+    earliest_config = {"earliest": 0}
+    latest = 1
+    latest_config = {"latest": 1}
+    mock_source = make_mock_source(mock_entrypoint_read, iter(
+        any_request_and_response_with_a_record() +
+        [
+            # here, we test that even if messages are emitted in a different order, we still rely on `emitted_at`
+            connector_configuration_control_message(latest, latest_config),
+            connector_configuration_control_message(earliest, earliest_config),
+        ]
+    )
+                                   )
+    connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
+    stream_read: StreamRead = connector_builder_handler.get_message_groups(
+        source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")
+    )
+
+    assert stream_read.latest_config_update == latest_config
+
+
+@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read')
+def test_given_multiple_control_messages_with_same_timestamp_then_stream_read_has_latest_based_on_message_order(mock_entrypoint_read):
+    emitted_at = 0
+    earliest_config = {"earliest": 0}
+    latest_config = {"latest": 1}
+    mock_source = make_mock_source(mock_entrypoint_read, iter(
+        any_request_and_response_with_a_record() +
+        [
+            connector_configuration_control_message(emitted_at, earliest_config),
+            connector_configuration_control_message(emitted_at, latest_config),
+        ]
+    )
+                                   )
+    connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
+    stream_read: StreamRead = connector_builder_handler.get_message_groups(
+        source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")
+    )
+
+    assert stream_read.latest_config_update == latest_config
+
+
 def make_mock_source(mock_entrypoint_read, return_value: Iterator) -> MagicMock:
     mock_source = MagicMock()
     mock_entrypoint_read.return_value = return_value
@@ -550,3 +628,22 @@ def record_message(stream: str, data: dict) -> AirbyteMessage:
 
 def slice_message(slice_descriptor: str = '{"key": "value"}') -> AirbyteMessage:
     return AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message="slice:" + slice_descriptor))
+
+
+def connector_configuration_control_message(emitted_at: float, config: dict) -> AirbyteMessage:
+    return AirbyteMessage(
+        type=MessageType.CONTROL,
+        control=AirbyteControlMessage(
+            type=OrchestratorType.CONNECTOR_CONFIG,
+            emitted_at=emitted_at,
+            connectorConfig=AirbyteControlConnectorConfigMessage(config=config),
+        )
+    )
+
+
+def any_request_and_response_with_a_record():
+    return [
+        request_log_message({"request": 1}),
+        response_log_message({"response": 2}),
+        record_message("hashiras", {"name": "Shinobu Kocho"}),
+    ]