airbyte-cdk 0.30.4__py3-none-any.whl → 0.31.1__py3-none-any.whl

{connector_builder → airbyte_cdk/connector_builder}/connector_builder_handler.py

@@ -7,6 +7,7 @@ from datetime import datetime
 from typing import Any, Dict, List, Mapping
 from urllib.parse import urljoin
 
+from airbyte_cdk.connector_builder.message_grouper import MessageGrouper
 from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, ConfiguredAirbyteCatalog
 from airbyte_cdk.models import Type
 from airbyte_cdk.models import Type as MessageType
@@ -15,7 +16,6 @@ from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from connector_builder.message_grouper import MessageGrouper
 
 DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE = 5
 DEFAULT_MAXIMUM_NUMBER_OF_SLICES = 5
@@ -31,13 +31,14 @@ def read_stream(source: DeclarativeSource, config: Mapping[str, Any], configured
         handler = MessageGrouper(max_pages_per_slice, max_slices)
         stream_name = configured_catalog.streams[0].stream.name  # The connector builder only supports a single stream
         stream_read = handler.get_message_groups(source, config, configured_catalog, max_records)
-        return AirbyteMessage(type=MessageType.RECORD, record=AirbyteRecordMessage(
-            data=dataclasses.asdict(stream_read),
-            stream=stream_name,
-            emitted_at=_emitted_at()
-        ))
+        return AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(data=dataclasses.asdict(stream_read), stream=stream_name, emitted_at=_emitted_at()),
+        )
     except Exception as exc:
-        error = AirbyteTracedException.from_exception(exc, message=f"Error reading stream with config={config} and catalog={configured_catalog}")
+        error = AirbyteTracedException.from_exception(
+            exc, message=f"Error reading stream with config={config} and catalog={configured_catalog}"
+        )
         return error.as_airbyte_message()
 
 
@@ -85,9 +86,7 @@ def _get_http_streams(source: ManifestDeclarativeSource, config: Dict[str, Any])
                     f"A declarative stream should only have a retriever of type HttpStream, but received: {stream.retriever.__class__}"
                 )
         else:
-            raise TypeError(
-                f"A declarative source should only contain streams of type DeclarativeStream, but received: {stream.__class__}"
-            )
+            raise TypeError(f"A declarative source should only contain streams of type DeclarativeStream, but received: {stream.__class__}")
     return http_streams
 
 

{connector_builder → airbyte_cdk/connector_builder}/main.py

@@ -7,11 +7,11 @@ import sys
 from typing import Any, List, Mapping, Optional, Tuple
 
 from airbyte_cdk.connector import BaseConnector
+from airbyte_cdk.connector_builder.connector_builder_handler import list_streams, read_stream, resolve_manifest
 from airbyte_cdk.entrypoint import AirbyteEntrypoint
 from airbyte_cdk.models import ConfiguredAirbyteCatalog
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from connector_builder.connector_builder_handler import list_streams, read_stream, resolve_manifest
 
 
 def create_source(config: Mapping[str, Any]) -> ManifestDeclarativeSource:

{connector_builder → airbyte_cdk/connector_builder}/message_grouper.py

@@ -9,11 +9,19 @@ from json import JSONDecodeError
 from typing import Any, Iterable, Iterator, Mapping, Optional, Union
 from urllib.parse import parse_qs, urlparse
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Type
+from airbyte_cdk.connector_builder.models import HttpRequest, HttpResponse, LogMessage, StreamRead, StreamReadPages, StreamReadSlices
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
+from airbyte_cdk.utils import AirbyteTracedException
 from airbyte_cdk.utils.schema_inferrer import SchemaInferrer
-from airbyte_protocol.models.airbyte_protocol import ConfiguredAirbyteCatalog
-from connector_builder.models import HttpRequest, HttpResponse, StreamRead, StreamReadPages, StreamReadSlices
+from airbyte_protocol.models.airbyte_protocol import (
+    AirbyteLogMessage,
+    AirbyteMessage,
+    AirbyteTraceMessage,
+    ConfiguredAirbyteCatalog,
+    Level,
+    TraceType,
+)
+from airbyte_protocol.models.airbyte_protocol import Type as MessageType
 
 
 class MessageGrouper:
@@ -24,12 +32,13 @@ class MessageGrouper:
         self._max_slices = max_slices
         self._max_record_limit = max_record_limit
 
-    def get_message_groups(self,
-                           source: DeclarativeSource,
-                           config: Mapping[str, Any],
-                           configured_catalog: ConfiguredAirbyteCatalog,
-                           record_limit: Optional[int] = None,
-                           ) -> StreamRead:
+    def get_message_groups(
+        self,
+        source: DeclarativeSource,
+        config: Mapping[str, Any],
+        configured_catalog: ConfiguredAirbyteCatalog,
+        record_limit: Optional[int] = None,
+    ) -> StreamRead:
         if record_limit is not None and not (1 <= record_limit <= 1000):
             raise ValueError(f"Record limit must be between 1 and 1000. Got {record_limit}")
         schema_inferrer = SchemaInferrer()
@@ -41,14 +50,18 @@ class MessageGrouper:
 
         slices = []
         log_messages = []
-        state = {}  # No support for incremental sync
         for message_group in self._get_message_groups(
-                source.read(self.logger, config, configured_catalog, state),
+                self._read_stream(source, config, configured_catalog),
                 schema_inferrer,
                 record_limit,
         ):
             if isinstance(message_group, AirbyteLogMessage):
-                log_messages.append({"message": message_group.message})
+                log_messages.append(LogMessage(**{"message": message_group.message, "level": message_group.level.value}))
+            elif isinstance(message_group, AirbyteTraceMessage):
+                if message_group.type == TraceType.ERROR:
+                    error_message = f"{message_group.error.message} - {message_group.error.stack_trace}"
+                    log_messages.append(LogMessage(**{"message": error_message, "level": "ERROR"}))
+
             else:
                 slices.append(message_group)
 
@@ -56,12 +69,14 @@ class MessageGrouper:
             logs=log_messages,
             slices=slices,
             test_read_limit_reached=self._has_reached_limit(slices),
-            inferred_schema=schema_inferrer.get_stream_schema(configured_catalog.streams[0].stream.name)  # The connector builder currently only supports reading from a single stream at a time
+            inferred_schema=schema_inferrer.get_stream_schema(
+                configured_catalog.streams[0].stream.name
+            ),  # The connector builder currently only supports reading from a single stream at a time
         )
 
     def _get_message_groups(
             self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, limit: int
-    ) -> Iterable[Union[StreamReadPages, AirbyteLogMessage]]:
+    ) -> Iterable[Union[StreamReadPages, AirbyteLogMessage, AirbyteTraceMessage]]:
         """
         Message groups are partitioned according to when request log messages are received. Subsequent response log messages
         and record messages belong to the prior request log message and when we encounter another request, append the latest
@@ -83,44 +98,56 @@ class MessageGrouper:
         current_slice_pages = []
         current_page_request: Optional[HttpRequest] = None
         current_page_response: Optional[HttpResponse] = None
+        had_error = False
 
         while records_count < limit and (message := next(messages, None)):
             if self._need_to_close_page(at_least_one_page_in_group, message):
-                self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records)
+                self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records, True)
                 current_page_request = None
                 current_page_response = None
 
-            if at_least_one_page_in_group and message.type == Type.LOG and message.log.message.startswith("slice:"):
+            if at_least_one_page_in_group and message.type == MessageType.LOG and message.log.message.startswith("slice:"):
                 yield StreamReadSlices(pages=current_slice_pages)
                 current_slice_pages = []
                 at_least_one_page_in_group = False
-            elif message.type == Type.LOG and message.log.message.startswith("request:"):
+            elif message.type == MessageType.LOG and message.log.message.startswith("request:"):
                 if not at_least_one_page_in_group:
                     at_least_one_page_in_group = True
                 current_page_request = self._create_request_from_log_message(message.log)
-            elif message.type == Type.LOG and message.log.message.startswith("response:"):
+            elif message.type == MessageType.LOG and message.log.message.startswith("response:"):
                 current_page_response = self._create_response_from_log_message(message.log)
-            elif message.type == Type.LOG:
+            elif message.type == MessageType.LOG:
+                if message.log.level == Level.ERROR:
+                    had_error = True
                 yield message.log
-            elif message.type == Type.RECORD:
+            elif message.type == MessageType.TRACE:
+                if message.trace.type == TraceType.ERROR:
+                    had_error = True
+                    yield message.trace
+            elif message.type == MessageType.RECORD:
                 current_page_records.append(message.record.data)
                 records_count += 1
                 schema_inferrer.accumulate(message.record)
         else:
-            self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records)
+            self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records, validate_page_complete=not had_error)
             yield StreamReadSlices(pages=current_slice_pages)
 
     @staticmethod
-    def _need_to_close_page(at_least_one_page_in_group, message):
+    def _need_to_close_page(at_least_one_page_in_group, message) -> bool:
         return (
-                at_least_one_page_in_group
-                and message.type == Type.LOG
-                and (message.log.message.startswith("request:") or message.log.message.startswith("slice:"))
+            at_least_one_page_in_group
+            and message.type == MessageType.LOG
+            and (message.log.message.startswith("request:") or message.log.message.startswith("slice:"))
         )
 
     @staticmethod
-    def _close_page(current_page_request, current_page_response, current_slice_pages, current_page_records):
-        if not current_page_request or not current_page_response:
+    def _close_page(current_page_request, current_page_response, current_slice_pages, current_page_records, validate_page_complete: bool):
+        """
+        Close a page when parsing message groups
+        @param validate_page_complete: in some cases, we expect the CDK to not return a response. As of today, this will only happen before
+        an uncaught exception and therefore, the assumption is that `validate_page_complete=True` only on the last page that is being closed
+        """
+        if validate_page_complete and (not current_page_request or not current_page_response):
             raise ValueError("Every message grouping should have at least one request and response")
 
         current_slice_pages.append(
@@ -128,6 +155,15 @@ class MessageGrouper:
         )
         current_page_records.clear()
 
+    def _read_stream(self, source, config, configured_catalog) -> Iterator[AirbyteMessage]:
+        # the generator can raise an exception
+        # iterate over the generated messages. if next raise an exception, catch it and yield it as an AirbyteLogMessage
+        try:
+            yield from source.read(logger=self.logger, config=config, catalog=configured_catalog, state={})
+        except Exception as e:
+            error_message = f"{e.args[0] if len(e.args) > 0 else str(e)}"
+            yield AirbyteTracedException.from_exception(e, message=error_message).as_airbyte_message()
+
     def _create_request_from_log_message(self, log_message: AirbyteLogMessage) -> Optional[HttpRequest]:
         # TODO: As a temporary stopgap, the CDK emits request data as a log message string. Ideally this should come in the
         # form of a custom message object defined in the Airbyte protocol, but this unblocks us in the immediate while the

{connector_builder → airbyte_cdk/connector_builder}/models.py

@@ -50,9 +50,15 @@ class StreamReadSlicesInner:
     state: Optional[Dict[str, Any]]
 
 
+@dataclass
+class LogMessage:
+    message: str
+    level: str
+
+
 @dataclass
 class StreamRead(object):
-    logs: List[object]
+    logs: List[LogMessage]
     slices: List[StreamReadSlicesInner]
     test_read_limit_reached: bool
     inferred_schema: Optional[Dict[str, Any]]

{airbyte_cdk-0.30.4.dist-info → airbyte_cdk-0.31.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.30.4
+Version: 0.31.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.30.4.dist-info → airbyte_cdk-0.31.1.dist-info}/RECORD

@@ -5,6 +5,11 @@ airbyte_cdk/entrypoint.py,sha256=9WkwB7Giqsz-VCaRkqCwrVIH9OfvULQs3-kz8IBfOh4,690
 airbyte_cdk/exception_handler.py,sha256=CwkiPdZ1WMOr3CBkvKFyHiyLerXGRqBrVlB4p0OImGI,1125
 airbyte_cdk/logger.py,sha256=4Mi2MEQi1uh59BP9Dxw_UEbZuxaJewqK_jvEU2b10nk,3985
 airbyte_cdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
+airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=9iG2hmaJyBtLDtdwtQp8M19pG2HQjR7Yy6jnBDggXMk,4160
+airbyte_cdk/connector_builder/main.py,sha256=IOijgSQ4A9KhqJplHSVPTrxH_cyGWW_9uNmSIwBo0l0,3021
+airbyte_cdk/connector_builder/message_grouper.py,sha256=-0AYh21eZeNJyyFPvSB84NNKMZznSX7bf4Ygi3deWEU,10972
+airbyte_cdk/connector_builder/models.py,sha256=y0PJ-LwJk3e1RzRmMfjQSBP9ENx_a0wBcWNCjlW72Ks,1832
 airbyte_cdk/destinations/__init__.py,sha256=0Uxmz3iBAyZJdk_bqUVt2pb0UwRTpFjTnFE6fQFbWKY,126
 airbyte_cdk/destinations/destination.py,sha256=_tIMnKcRQbtIsjVvNOVjfbIxgCNLuBXQwQj8MyVm3BI,5420
 airbyte_cdk/models/__init__.py,sha256=LPQcYdDPwrCXiBPe_jexO4UAcbovIb1V9tHB6I7Un30,633
@@ -152,16 +157,11 @@ airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=q3aDl8T10ufGbeqnUPqbZLxQcHdkf2
 airbyte_cdk/utils/event_timing.py,sha256=Hn5kCc9xGKLcV5EYpJCZwNiz9neKKu2WG8FJF_hy278,2377
 airbyte_cdk/utils/schema_inferrer.py,sha256=LQLOlraFksg7_sqpJNhy9pS_K42GVxG634ogM_P2s5E,2361
 airbyte_cdk/utils/traced_exception.py,sha256=9G2sG9eYkvn6Aa7rMuUW_KIRszRaTc_xdnTQNDKyKGI,3216
-connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-connector_builder/connector_builder_handler.py,sha256=QF0Rus1_NhKNfma8EtL6sVWSu8kySLDj2m5IcZ-9OsU,4167
-connector_builder/main.py,sha256=6jnHUX5pVG3dVPC6D4oMGneoxzmbpBqZNB-rfj7_Mjc,3009
-connector_builder/message_grouper.py,sha256=NkiDNvwXkP4EB-501hGpmZZCJjaJo7xYdGD2xmiGoH8,9121
-connector_builder/models.py,sha256=mf9cAZaxFW7uFp0gyHiJpuGlAuGuq2zFvs76VMYqUkA,1765
 source_declarative_manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 source_declarative_manifest/main.py,sha256=HXzuRsRyhHwPrGU-hc4S7RrgoOoHImqkdfbmO2geBeE,1027
 unit_tests/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-unit_tests/connector_builder/test_connector_builder_handler.py,sha256=kuw1QAIsGt8-R24Q7JkjsE6ErOy6HaDpPzIbK0d5KRk,18730
-unit_tests/connector_builder/test_message_grouper.py,sha256=1yOFy9Umr7Y1rIQN0Rvt6nHdDzdkmmncxoBD5PkRBr4,22623
+unit_tests/connector_builder/test_connector_builder_handler.py,sha256=fwOlQPdc9quPUoZYXg_GVXqdkw5TLWs0CuBjYEMAcfM,19745
+unit_tests/connector_builder/test_message_grouper.py,sha256=1_DXyuqHmr8B7b1t8PSpmYWaHvYYAB2mMwpp_YAbYwc,22959
 unit_tests/connector_builder/utils.py,sha256=AAggdGWP-mNuWOZUHLAVIbjTeIcdPo-3pbMm5zdYpS0,796
 unit_tests/destinations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/destinations/test_destination.py,sha256=koG_j812KMkcIxoUH6XlAL3zsephZJmlHvyzJXm0dCs,10269
@@ -257,8 +257,8 @@ unit_tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 unit_tests/utils/test_schema_inferrer.py,sha256=ckl17GlNOZInqgxni7Z2A0bg_p6JDy0GVFAG8ph67pw,3288
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.30.4.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.30.4.dist-info/METADATA,sha256=RIkRi-qItoUoeznoJcGPfwmobM39pO2okwNIp-OHadA,8902
-airbyte_cdk-0.30.4.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-airbyte_cdk-0.30.4.dist-info/top_level.txt,sha256=1QIVQ1LzAH7ad20nfhHPxy0-7QJU22jfCrLJSLuaFog,69
-airbyte_cdk-0.30.4.dist-info/RECORD,,
+airbyte_cdk-0.31.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.31.1.dist-info/METADATA,sha256=nSfqOPf-MueD0MHg9bTQU8PWrhP43lmxd6JK1IDNj-M,8902
+airbyte_cdk-0.31.1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+airbyte_cdk-0.31.1.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.31.1.dist-info/RECORD,,

{airbyte_cdk-0.30.4.dist-info → airbyte_cdk-0.31.1.dist-info}/top_level.txt

@@ -1,4 +1,3 @@
 airbyte_cdk
-connector_builder
 source_declarative_manifest
 unit_tests

unit_tests/connector_builder/test_connector_builder_handler.py

@@ -3,21 +3,22 @@
 #
 
 import copy
+import dataclasses
 import json
 from unittest import mock
 from unittest.mock import patch
 
-import connector_builder
 import pytest
+from airbyte_cdk import connector_builder
+from airbyte_cdk.connector_builder.connector_builder_handler import list_streams, resolve_manifest
+from airbyte_cdk.connector_builder.main import handle_connector_builder_request, handle_request, read_stream
+from airbyte_cdk.connector_builder.models import LogMessage, StreamRead, StreamReadSlicesInner, StreamReadSlicesInnerPagesInner
 from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, ConfiguredAirbyteCatalog
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from airbyte_cdk.sources.streams.core import Stream
 from airbyte_cdk.sources.streams.http import HttpStream
-from connector_builder.connector_builder_handler import list_streams, resolve_manifest
-from connector_builder.main import handle_connector_builder_request, handle_request, read_stream
-from connector_builder.models import StreamRead, StreamReadSlicesInner, StreamReadSlicesInnerPagesInner
 from unit_tests.connector_builder.utils import create_configured_catalog
 
 _stream_name = "stream_with_custom_requester"
@@ -215,7 +216,7 @@ def test_resolve_manifest(valid_resolve_manifest_config_file):
                             "primary_key": _stream_primary_key,
                             "url_base": _stream_url_base,
                             "$parameters": _stream_options,
-                            "page_size": 10
+                            "page_size": 10,
                         },
                         "name": _stream_name,
                         "primary_key": _stream_primary_key,
@@ -315,7 +316,7 @@ def test_read():
             emitted_at=1,
         ),
     )
-    with patch("connector_builder.message_grouper.MessageGrouper.get_message_groups", return_value=stream_read):
+    with patch("airbyte_cdk.connector_builder.message_grouper.MessageGrouper.get_message_groups", return_value=stream_read):
         output_record = handle_connector_builder_request(
             source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG)
         )
@@ -323,14 +324,31 @@ def test_read():
         assert output_record == expected_airbyte_message
 
 
-def test_read_returns_error_response():
+@patch("traceback.TracebackException.from_exception")
+def test_read_returns_error_response(mock_from_exception):
     class MockManifestDeclarativeSource:
         def read(self, logger, config, catalog, state):
-            raise ValueError
+            raise ValueError("error_message")
+
+    stack_trace = "a stack trace"
+    mock_from_exception.return_value = stack_trace
 
     source = MockManifestDeclarativeSource()
     response = read_stream(source, TEST_READ_CONFIG, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG))
-    assert "Error reading" in response.trace.error.message
+
+    expected_stream_read = StreamRead(logs=[LogMessage("error_message - a stack trace", "ERROR")],
+                                      slices=[StreamReadSlicesInner(
+                                          pages=[StreamReadSlicesInnerPagesInner(records=[], request=None, response=None)],
+                                          slice_descriptor=None, state=None)],
+                                      test_read_limit_reached=False,
+                                      inferred_schema=None)
+
+    expected_message = AirbyteMessage(
+        type=MessageType.RECORD,
+        record=AirbyteRecordMessage(stream=_stream_name, data=dataclasses.asdict(expected_stream_read), emitted_at=1),
+    )
+    response.record.emitted_at = 1
+    assert response == expected_message
 
 
 @pytest.mark.parametrize(
@@ -385,10 +403,12 @@ def test_list_streams(manifest_declarative_source):
 
     assert result.type == MessageType.RECORD
     assert result.record.stream == "list_streams"
-    assert result.record.data == {"streams": [
-        {"name": "a name", "url": "https://a-url-base.com/a-path"},
-        {"name": "another name", "url": "https://another-url-base.com/another-path"}
-    ]}
+    assert result.record.data == {
+        "streams": [
+            {"name": "a name", "url": "https://a-url-base.com/a-path"},
+            {"name": "another name", "url": "https://another-url-base.com/another-path"},
+        ]
+    }
 
 
 def test_given_stream_is_not_declarative_stream_when_list_streams_then_return_exception_message(manifest_declarative_source):

unit_tests/connector_builder/test_message_grouper.py

@@ -7,17 +7,17 @@ from typing import Iterator
 from unittest.mock import MagicMock
 
 import pytest
+from airbyte_cdk.connector_builder.message_grouper import MessageGrouper
+from airbyte_cdk.connector_builder.models import HttpRequest, HttpResponse, LogMessage, StreamRead, StreamReadPages
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteRecordMessage, Level
 from airbyte_cdk.models import Type as MessageType
-from connector_builder.message_grouper import MessageGrouper
-from connector_builder.models import HttpRequest, HttpResponse, StreamRead, StreamReadPages
 from unit_tests.connector_builder.utils import create_configured_catalog
 
 MAX_PAGES_PER_SLICE = 4
 MAX_SLICES = 3
 
 MANIFEST = {
-    "version": "0.1.0",
+    "version": "0.30.0",
     "type": "DeclarativeSource",
     "definitions": {
         "selector": {"extractor": {"field_path": ["items"], "type": "DpathExtractor"}, "type": "RecordSelector"},
@@ -124,8 +124,9 @@ def test_get_grouped_messages():
     )
 
     connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
-    actual_response: StreamRead = connector_builder_handler.get_message_groups(source=mock_source, config=CONFIG,
-                                                                               configured_catalog=create_configured_catalog("hashiras"))
+    actual_response: StreamRead = connector_builder_handler.get_message_groups(
+        source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")
+    )
     assert actual_response.inferred_schema == expected_schema
 
     single_slice = actual_response.slices[0]
@@ -166,9 +167,9 @@ def test_get_grouped_messages_with_logs():
         ),
     ]
     expected_logs = [
-        {"message": "log message before the request"},
-        {"message": "log message during the page"},
-        {"message": "log message after the response"},
+        LogMessage(**{"message": "log message before the request", "level": "INFO"}),
+        LogMessage(**{"message": "log message during the page", "level": "INFO"}),
+        LogMessage(**{"message": "log message after the response", "level": "INFO"}),
     ]
 
     mock_source = make_mock_source(
@@ -187,8 +188,9 @@ def test_get_grouped_messages_with_logs():
 
     connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
 
-    actual_response: StreamRead = connector_builder_handler.get_message_groups(source=mock_source, config=CONFIG,
-                                                                               configured_catalog=create_configured_catalog("hashiras"))
+    actual_response: StreamRead = connector_builder_handler.get_message_groups(
+        source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")
+    )
     single_slice = actual_response.slices[0]
     for i, actual_page in enumerate(single_slice.pages):
         assert actual_page == expected_pages[i]
@@ -229,9 +231,9 @@ def test_get_grouped_messages_record_limit(request_record_limit, max_record_limi
     record_limit = min(request_record_limit, max_record_limit)
 
     api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES, max_record_limit=max_record_limit)
-    actual_response: StreamRead = api.get_message_groups(mock_source, config=CONFIG,
-                                                         configured_catalog=create_configured_catalog("hashiras"),
-                                                         record_limit=request_record_limit)
+    actual_response: StreamRead = api.get_message_groups(
+        mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), record_limit=request_record_limit
+    )
     single_slice = actual_response.slices[0]
     total_records = 0
     for i, actual_page in enumerate(single_slice.pages):
@@ -270,8 +272,9 @@ def test_get_grouped_messages_default_record_limit(max_record_limit):
     n_records = 2
 
     api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES, max_record_limit=max_record_limit)
-    actual_response: StreamRead = api.get_message_groups(source=mock_source, config=CONFIG,
-                                                         configured_catalog=create_configured_catalog("hashiras"))
+    actual_response: StreamRead = api.get_message_groups(
+        source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")
+    )
     single_slice = actual_response.slices[0]
     total_records = 0
     for i, actual_page in enumerate(single_slice.pages):
@@ -352,8 +355,9 @@ def test_get_grouped_messages_no_records():
 
     message_grouper = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
 
-    actual_response: StreamRead = message_grouper.get_message_groups(source=mock_source, config=CONFIG,
-                                                                     configured_catalog=create_configured_catalog("hashiras"))
+    actual_response: StreamRead = message_grouper.get_message_groups(
+        source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")
+    )
 
     single_slice = actual_response.slices[0]
     for i, actual_page in enumerate(single_slice.pages):
@@ -459,8 +463,9 @@ def test_get_grouped_messages_with_many_slices():
 
     connecto_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
 
-    stream_read: StreamRead = connecto_builder_handler.get_message_groups(source=mock_source, config=CONFIG,
-                                                                          configured_catalog=create_configured_catalog("hashiras"))
+    stream_read: StreamRead = connecto_builder_handler.get_message_groups(
+        source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")
+    )
 
     assert not stream_read.test_read_limit_reached
     assert len(stream_read.slices) == 2
@@ -484,8 +489,9 @@ def test_get_grouped_messages_given_maximum_number_of_slices_then_test_read_limi
 
     api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
 
-    stream_read: StreamRead = api.get_message_groups(source=mock_source, config=CONFIG,
-                                                     configured_catalog=create_configured_catalog("hashiras"))
+    stream_read: StreamRead = api.get_message_groups(
+        source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")
+    )
 
     assert stream_read.test_read_limit_reached
 
@@ -500,12 +506,28 @@ def test_get_grouped_messages_given_maximum_number_of_pages_then_test_read_limit
 
     api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
 
-    stream_read: StreamRead = api.get_message_groups(source=mock_source, config=CONFIG,
-                                                     configured_catalog=create_configured_catalog("hashiras"))
+    stream_read: StreamRead = api.get_message_groups(
+        source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")
+    )
 
     assert stream_read.test_read_limit_reached
 
 
+def test_read_stream_returns_error_if_stream_does_not_exist():
+    mock_source = MagicMock()
+    mock_source.read.side_effect = ValueError("error")
+
+    full_config = {**CONFIG, **{"__injected_declarative_manifest": MANIFEST}}
+
+    message_grouper = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES)
+    actual_response = message_grouper.get_message_groups(source=mock_source, config=full_config,
+                                                         configured_catalog=create_configured_catalog("not_in_manifest"))
+
+    assert 1 == len(actual_response.logs)
+    assert "Traceback" in actual_response.logs[0].message
+    assert "ERROR" in actual_response.logs[0].level
+
+
 def make_mock_source(return_value: Iterator) -> MagicMock:
     mock_source = MagicMock()
     mock_source.read.return_value = return_value