airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

airbyte_cdk/connector_builder/message_grouper.py

@@ -7,7 +7,6 @@ import logging
 from copy import deepcopy
 from json import JSONDecodeError
 from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Union
-from urllib.parse import parse_qs, urlparse
 
 from airbyte_cdk.connector_builder.models import (
     AuxiliaryRequest,
@@ -19,22 +18,23 @@ from airbyte_cdk.connector_builder.models import (
     StreamReadSlices,
 )
 from airbyte_cdk.entrypoint import AirbyteEntrypoint
-from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
-from airbyte_cdk.sources.utils.slice_logger import SliceLogger
-from airbyte_cdk.sources.utils.types import JsonType
-from airbyte_cdk.utils import AirbyteTracedException
-from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer
-from airbyte_cdk.utils.schema_inferrer import SchemaInferrer
-from airbyte_protocol.models.airbyte_protocol import (
+from airbyte_cdk.models import (
     AirbyteControlMessage,
     AirbyteLogMessage,
     AirbyteMessage,
+    AirbyteStateMessage,
     AirbyteTraceMessage,
     ConfiguredAirbyteCatalog,
     OrchestratorType,
     TraceType,
 )
-from airbyte_protocol.models.airbyte_protocol import Type as MessageType
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
+from airbyte_cdk.sources.utils.slice_logger import SliceLogger
+from airbyte_cdk.sources.utils.types import JsonType
+from airbyte_cdk.utils import AirbyteTracedException
+from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer
+from airbyte_cdk.utils.schema_inferrer import SchemaInferrer, SchemaValidationException
 
 
 class MessageGrouper:
@@ -45,16 +45,55 @@ class MessageGrouper:
         self._max_slices = max_slices
         self._max_record_limit = max_record_limit
 
+    def _pk_to_nested_and_composite_field(
+        self, field: Optional[Union[str, List[str], List[List[str]]]]
+    ) -> List[List[str]]:
+        if not field:
+            return [[]]
+
+        if isinstance(field, str):
+            return [[field]]
+
+        is_composite_key = isinstance(field[0], str)
+        if is_composite_key:
+            return [[i] for i in field]  # type: ignore  # the type of field is expected to be List[str] here
+
+        return field  # type: ignore  # the type of field is expected to be List[List[str]] here
+
+    def _cursor_field_to_nested_and_composite_field(
+        self, field: Union[str, List[str]]
+    ) -> List[List[str]]:
+        if not field:
+            return [[]]
+
+        if isinstance(field, str):
+            return [[field]]
+
+        is_nested_key = isinstance(field[0], str)
+        if is_nested_key:
+            return [field]
+
+        raise ValueError(f"Unknown type for cursor field `{field}")
+
     def get_message_groups(
         self,
         source: DeclarativeSource,
         config: Mapping[str, Any],
         configured_catalog: ConfiguredAirbyteCatalog,
+        state: List[AirbyteStateMessage],
         record_limit: Optional[int] = None,
     ) -> StreamRead:
         if record_limit is not None and not (1 <= record_limit <= self._max_record_limit):
-            raise ValueError(f"Record limit must be between 1 and {self._max_record_limit}. Got {record_limit}")
-        schema_inferrer = SchemaInferrer()
+            raise ValueError(
+                f"Record limit must be between 1 and {self._max_record_limit}. Got {record_limit}"
+            )
+        stream = source.streams(config)[
+            0
+        ]  # The connector builder currently only supports reading from a single stream at a time
+        schema_inferrer = SchemaInferrer(
+            self._pk_to_nested_and_composite_field(stream.primary_key),
+            self._cursor_field_to_nested_and_composite_field(stream.cursor_field),
+        )
         datetime_format_inferrer = DatetimeFormatInferrer()
 
         if record_limit is None:
@@ -67,19 +106,34 @@ class MessageGrouper:
         latest_config_update: AirbyteControlMessage = None
         auxiliary_requests = []
         for message_group in self._get_message_groups(
-            self._read_stream(source, config, configured_catalog),
+            self._read_stream(source, config, configured_catalog, state),
             schema_inferrer,
             datetime_format_inferrer,
             record_limit,
         ):
             if isinstance(message_group, AirbyteLogMessage):
-                log_messages.append(LogMessage(**{"message": message_group.message, "level": message_group.level.value}))
+                log_messages.append(
+                    LogMessage(
+                        **{"message": message_group.message, "level": message_group.level.value}
+                    )
+                )
             elif isinstance(message_group, AirbyteTraceMessage):
                 if message_group.type == TraceType.ERROR:
-                    error_message = f"{message_group.error.message} - {message_group.error.stack_trace}"
-                    log_messages.append(LogMessage(**{"message": error_message, "level": "ERROR"}))
+                    log_messages.append(
+                        LogMessage(
+                            **{
+                                "message": message_group.error.message,
+                                "level": "ERROR",
+                                "internal_message": message_group.error.internal_message,
+                                "stacktrace": message_group.error.stack_trace,
+                            }
+                        )
+                    )
             elif isinstance(message_group, AirbyteControlMessage):
-                if not latest_config_update or latest_config_update.emitted_at <= message_group.emitted_at:
+                if (
+                    not latest_config_update
+                    or latest_config_update.emitted_at <= message_group.emitted_at
+                ):
                     latest_config_update = message_group
             elif isinstance(message_group, AuxiliaryRequest):
                 auxiliary_requests.append(message_group)
@@ -88,15 +142,24 @@ class MessageGrouper:
             else:
                 raise ValueError(f"Unknown message group type: {type(message_group)}")
 
+        try:
+            # The connector builder currently only supports reading from a single stream at a time
+            configured_stream = configured_catalog.streams[0]
+            schema = schema_inferrer.get_stream_schema(configured_stream.stream.name)
+        except SchemaValidationException as exception:
+            for validation_error in exception.validation_errors:
+                log_messages.append(LogMessage(validation_error, "ERROR"))
+            schema = exception.schema
+
         return StreamRead(
             logs=log_messages,
             slices=slices,
             test_read_limit_reached=self._has_reached_limit(slices),
             auxiliary_requests=auxiliary_requests,
-            inferred_schema=schema_inferrer.get_stream_schema(
-                configured_catalog.streams[0].stream.name
-            ),  # The connector builder currently only supports reading from a single stream at a time
-            latest_config_update=self._clean_config(latest_config_update.connectorConfig.config) if latest_config_update else None,
+            inferred_schema=schema,
+            latest_config_update=self._clean_config(latest_config_update.connectorConfig.config)
+            if latest_config_update
+            else None,
             inferred_datetime_formats=datetime_format_inferrer.get_inferred_datetime_formats(),
         )
 
@@ -106,7 +169,15 @@ class MessageGrouper:
         schema_inferrer: SchemaInferrer,
         datetime_format_inferrer: DatetimeFormatInferrer,
         limit: int,
-    ) -> Iterable[Union[StreamReadPages, AirbyteControlMessage, AirbyteLogMessage, AirbyteTraceMessage, AuxiliaryRequest]]:
+    ) -> Iterable[
+        Union[
+            StreamReadPages,
+            AirbyteControlMessage,
+            AirbyteLogMessage,
+            AirbyteTraceMessage,
+            AuxiliaryRequest,
+        ]
+    ]:
         """
         Message groups are partitioned according to when request log messages are received. Subsequent response log messages
         and record messages belong to the prior request log message and when we encounter another request, append the latest
@@ -129,42 +200,64 @@ class MessageGrouper:
         current_slice_pages: List[StreamReadPages] = []
         current_page_request: Optional[HttpRequest] = None
         current_page_response: Optional[HttpResponse] = None
+        latest_state_message: Optional[Dict[str, Any]] = None
 
         while records_count < limit and (message := next(messages, None)):
             json_object = self._parse_json(message.log) if message.type == MessageType.LOG else None
             if json_object is not None and not isinstance(json_object, dict):
-                raise ValueError(f"Expected log message to be a dict, got {json_object} of type {type(json_object)}")
+                raise ValueError(
+                    f"Expected log message to be a dict, got {json_object} of type {type(json_object)}"
+                )
             json_message: Optional[Dict[str, JsonType]] = json_object
             if self._need_to_close_page(at_least_one_page_in_group, message, json_message):
-                self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records)
+                self._close_page(
+                    current_page_request,
+                    current_page_response,
+                    current_slice_pages,
+                    current_page_records,
+                )
                 current_page_request = None
                 current_page_response = None
 
             if (
                 at_least_one_page_in_group
                 and message.type == MessageType.LOG
-                and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX)
+                and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX)  # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message
             ):
-                yield StreamReadSlices(pages=current_slice_pages, slice_descriptor=current_slice_descriptor)
-                current_slice_descriptor = self._parse_slice_description(message.log.message)
+                yield StreamReadSlices(
+                    pages=current_slice_pages,
+                    slice_descriptor=current_slice_descriptor,
+                    state=[latest_state_message] if latest_state_message else [],
+                )
+                current_slice_descriptor = self._parse_slice_description(message.log.message)  # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message
                 current_slice_pages = []
                 at_least_one_page_in_group = False
-            elif message.type == MessageType.LOG and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX):
+            elif message.type == MessageType.LOG and message.log.message.startswith(  # type: ignore[union-attr] # None doesn't have 'message'
+                SliceLogger.SLICE_LOG_PREFIX
+            ):
                 # parsing the first slice
-                current_slice_descriptor = self._parse_slice_description(message.log.message)
+                current_slice_descriptor = self._parse_slice_description(message.log.message)  # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message
             elif message.type == MessageType.LOG:
                 if json_message is not None and self._is_http_log(json_message):
                     if self._is_auxiliary_http_request(json_message):
                         airbyte_cdk = json_message.get("airbyte_cdk", {})
                         if not isinstance(airbyte_cdk, dict):
-                            raise ValueError(f"Expected airbyte_cdk to be a dict, got {airbyte_cdk} of type {type(airbyte_cdk)}")
+                            raise ValueError(
+                                f"Expected airbyte_cdk to be a dict, got {airbyte_cdk} of type {type(airbyte_cdk)}"
+                            )
                         stream = airbyte_cdk.get("stream", {})
                         if not isinstance(stream, dict):
-                            raise ValueError(f"Expected stream to be a dict, got {stream} of type {type(stream)}")
-                        title_prefix = "Parent stream: " if stream.get("is_substream", False) else ""
+                            raise ValueError(
+                                f"Expected stream to be a dict, got {stream} of type {type(stream)}"
+                            )
+                        title_prefix = (
+                            "Parent stream: " if stream.get("is_substream", False) else ""
+                        )
                         http = json_message.get("http", {})
                         if not isinstance(http, dict):
-                            raise ValueError(f"Expected http to be a dict, got {http} of type {type(http)}")
+                            raise ValueError(
+                                f"Expected http to be a dict, got {http} of type {type(http)}"
+                            )
                         yield AuxiliaryRequest(
                             title=title_prefix + str(http.get("title", None)),
                             description=str(http.get("description", None)),
@@ -178,26 +271,47 @@ class MessageGrouper:
                 else:
                     yield message.log
             elif message.type == MessageType.TRACE:
-                if message.trace.type == TraceType.ERROR:
+                if message.trace.type == TraceType.ERROR:  # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has trace.type
                     yield message.trace
             elif message.type == MessageType.RECORD:
-                current_page_records.append(message.record.data)
+                current_page_records.append(message.record.data)  # type: ignore[arg-type, union-attr] # AirbyteMessage with MessageType.RECORD has record.data
                 records_count += 1
                 schema_inferrer.accumulate(message.record)
                 datetime_format_inferrer.accumulate(message.record)
-            elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG:
+            elif (
+                message.type == MessageType.CONTROL
+                and message.control.type == OrchestratorType.CONNECTOR_CONFIG  # type: ignore[union-attr] # None doesn't have 'type'
+            ):
                 yield message.control
+            elif message.type == MessageType.STATE:
+                latest_state_message = message.state  # type: ignore[assignment]
         else:
             if current_page_request or current_page_response or current_page_records:
-                self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records)
-                yield StreamReadSlices(pages=current_slice_pages, slice_descriptor=current_slice_descriptor)
+                self._close_page(
+                    current_page_request,
+                    current_page_response,
+                    current_slice_pages,
+                    current_page_records,
+                )
+                yield StreamReadSlices(
+                    pages=current_slice_pages,
+                    slice_descriptor=current_slice_descriptor,
+                    state=[latest_state_message] if latest_state_message else [],
+                )
 
     @staticmethod
-    def _need_to_close_page(at_least_one_page_in_group: bool, message: AirbyteMessage, json_message: Optional[Dict[str, Any]]) -> bool:
+    def _need_to_close_page(
+        at_least_one_page_in_group: bool,
+        message: AirbyteMessage,
+        json_message: Optional[Dict[str, Any]],
+    ) -> bool:
         return (
             at_least_one_page_in_group
             and message.type == MessageType.LOG
-            and (MessageGrouper._is_page_http_request(json_message) or message.log.message.startswith("slice:"))
+            and (
+                MessageGrouper._is_page_http_request(json_message)
+                or message.log.message.startswith("slice:")  # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message
+            )
         )
 
     @staticmethod
@@ -205,7 +319,9 @@ class MessageGrouper:
         if not json_message:
             return False
         else:
-            return MessageGrouper._is_http_log(json_message) and not MessageGrouper._is_auxiliary_http_request(json_message)
+            return MessageGrouper._is_http_log(
+                json_message
+            ) and not MessageGrouper._is_auxiliary_http_request(json_message)
 
     @staticmethod
     def _is_http_log(message: Dict[str, JsonType]) -> bool:
@@ -236,20 +352,44 @@ class MessageGrouper:
         Close a page when parsing message groups
         """
         current_slice_pages.append(
-            StreamReadPages(request=current_page_request, response=current_page_response, records=deepcopy(current_page_records))  # type: ignore
+            StreamReadPages(
+                request=current_page_request,
+                response=current_page_response,
+                records=deepcopy(current_page_records),  # type: ignore [arg-type]
+            )
         )
         current_page_records.clear()
 
     def _read_stream(
-        self, source: DeclarativeSource, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog
+        self,
+        source: DeclarativeSource,
+        config: Mapping[str, Any],
+        configured_catalog: ConfiguredAirbyteCatalog,
+        state: List[AirbyteStateMessage],
     ) -> Iterator[AirbyteMessage]:
         # the generator can raise an exception
         # iterate over the generated messages. if next raise an exception, catch it and yield it as an AirbyteLogMessage
         try:
-            yield from AirbyteEntrypoint(source).read(source.spec(self.logger), config, configured_catalog, {})
+            yield from AirbyteEntrypoint(source).read(
+                source.spec(self.logger), config, configured_catalog, state
+            )
+        except AirbyteTracedException as traced_exception:
+            # Look for this message which indicates that it is the "final exception" raised by AbstractSource.
+            # If it matches, don't yield this as we don't need to show this in the Builder.
+            # This is somewhat brittle as it relies on the message string, but if they drift then the worst case
+            # is that this message will be shown in the Builder.
+            if (
+                traced_exception.message is not None
+                and "During the sync, the following streams did not sync successfully"
+                in traced_exception.message
+            ):
+                return
+            yield traced_exception.as_airbyte_message()
         except Exception as e:
             error_message = f"{e.args[0] if len(e.args) > 0 else str(e)}"
-            yield AirbyteTracedException.from_exception(e, message=error_message).as_airbyte_message()
+            yield AirbyteTracedException.from_exception(
+                e, message=error_message
+            ).as_airbyte_message()
 
     @staticmethod
     def _parse_json(log_message: AirbyteLogMessage) -> JsonType:
@@ -264,15 +404,12 @@ class MessageGrouper:
 
     @staticmethod
     def _create_request_from_log_message(json_http_message: Dict[str, Any]) -> HttpRequest:
-        url = urlparse(json_http_message.get("url", {}).get("full", ""))
-        full_path = f"{url.scheme}://{url.hostname}{url.path}" if url else ""
+        url = json_http_message.get("url", {}).get("full", "")
         request = json_http_message.get("http", {}).get("request", {})
-        parameters = parse_qs(url.query) or None
         return HttpRequest(
-            url=full_path,
+            url=url,
             http_method=request.get("method", ""),
             headers=request.get("headers"),
-            parameters=parameters,
             body=request.get("body", {}).get("content", ""),
         )
 
@@ -280,7 +417,9 @@ class MessageGrouper:
     def _create_response_from_log_message(json_http_message: Dict[str, Any]) -> HttpResponse:
         response = json_http_message.get("http", {}).get("response", {})
         body = response.get("body", {}).get("content", "")
-        return HttpResponse(status=response.get("status_code"), body=body, headers=response.get("headers"))
+        return HttpResponse(
+            status=response.get("status_code"), body=body, headers=response.get("headers")
+        )
 
     def _has_reached_limit(self, slices: List[StreamReadSlices]) -> bool:
         if len(slices) >= self._max_slices:

airbyte_cdk/connector_builder/models.py

@@ -16,7 +16,6 @@ class HttpResponse:
 @dataclass
 class HttpRequest:
     url: str
-    parameters: Optional[Dict[str, Any]]
     headers: Optional[Dict[str, Any]]
     http_method: str
     body: Optional[str] = None
@@ -33,13 +32,15 @@ class StreamReadPages:
 class StreamReadSlices:
     pages: List[StreamReadPages]
     slice_descriptor: Optional[Dict[str, Any]]
-    state: Optional[Dict[str, Any]] = None
+    state: Optional[List[Dict[str, Any]]] = None
 
 
 @dataclass
 class LogMessage:
     message: str
     level: str
+    internal_message: Optional[str] = None
+    stacktrace: Optional[str] = None
 
 
 @dataclass

airbyte_cdk/destinations/__init__.py

@@ -1,7 +1,8 @@
-#
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#
+"""The destinations module provides classes for building destination connectors."""
 
 from .destination import Destination
 
-__all__ = ["Destination"]
+__all__ = [
+    "Destination",
+]

airbyte_cdk/destinations/destination.py

@@ -9,12 +9,19 @@ import sys
 from abc import ABC, abstractmethod
 from typing import Any, Iterable, List, Mapping
 
+import orjson
+
 from airbyte_cdk.connector import Connector
 from airbyte_cdk.exception_handler import init_uncaught_exception_handler
-from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteMessageSerializer,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteCatalogSerializer,
+    Type,
+)
 from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from pydantic import ValidationError
 
 logger = logging.getLogger("airbyte")
 
@@ -24,7 +31,10 @@ class Destination(Connector, ABC):
 
     @abstractmethod
     def write(
-        self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage]
+        self,
+        config: Mapping[str, Any],
+        configured_catalog: ConfiguredAirbyteCatalog,
+        input_messages: Iterable[AirbyteMessage],
     ) -> Iterable[AirbyteMessage]:
         """Implement to define how the connector writes data to the destination"""
 
@@ -36,17 +46,26 @@ class Destination(Connector, ABC):
         """Reads from stdin, converting to Airbyte messages"""
         for line in input_stream:
             try:
-                yield AirbyteMessage.parse_raw(line)
-            except ValidationError:
-                logger.info(f"ignoring input which can't be deserialized as Airbyte Message: {line}")
+                yield AirbyteMessageSerializer.load(orjson.loads(line))
+            except orjson.JSONDecodeError:
+                logger.info(
+                    f"ignoring input which can't be deserialized as Airbyte Message: {line}"
+                )
 
     def _run_write(
-        self, config: Mapping[str, Any], configured_catalog_path: str, input_stream: io.TextIOWrapper
+        self,
+        config: Mapping[str, Any],
+        configured_catalog_path: str,
+        input_stream: io.TextIOWrapper,
     ) -> Iterable[AirbyteMessage]:
-        catalog = ConfiguredAirbyteCatalog.parse_file(configured_catalog_path)
+        catalog = ConfiguredAirbyteCatalogSerializer.load(
+            orjson.loads(open(configured_catalog_path).read())
+        )
         input_messages = self._parse_input_stream(input_stream)
         logger.info("Begin writing to the destination...")
-        yield from self.write(config=config, configured_catalog=catalog, input_messages=input_messages)
+        yield from self.write(
+            config=config, configured_catalog=catalog, input_messages=input_messages
+        )
         logger.info("Writing complete.")
 
     def parse_args(self, args: List[str]) -> argparse.Namespace:
@@ -60,18 +79,30 @@ class Destination(Connector, ABC):
         subparsers = main_parser.add_subparsers(title="commands", dest="command")
 
         # spec
-        subparsers.add_parser("spec", help="outputs the json configuration specification", parents=[parent_parser])
+        subparsers.add_parser(
+            "spec", help="outputs the json configuration specification", parents=[parent_parser]
+        )
 
         # check
-        check_parser = subparsers.add_parser("check", help="checks the config can be used to connect", parents=[parent_parser])
+        check_parser = subparsers.add_parser(
+            "check", help="checks the config can be used to connect", parents=[parent_parser]
+        )
         required_check_parser = check_parser.add_argument_group("required named arguments")
-        required_check_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file")
+        required_check_parser.add_argument(
+            "--config", type=str, required=True, help="path to the json configuration file"
+        )
 
         # write
-        write_parser = subparsers.add_parser("write", help="Writes data to the destination", parents=[parent_parser])
+        write_parser = subparsers.add_parser(
+            "write", help="Writes data to the destination", parents=[parent_parser]
+        )
         write_required = write_parser.add_argument_group("required named arguments")
-        write_required.add_argument("--config", type=str, required=True, help="path to the JSON configuration file")
-        write_required.add_argument("--catalog", type=str, required=True, help="path to the configured catalog JSON file")
+        write_required.add_argument(
+            "--config", type=str, required=True, help="path to the JSON configuration file"
+        )
+        write_required.add_argument(
+            "--catalog", type=str, required=True, help="path to the configured catalog JSON file"
+        )
 
         parsed_args = main_parser.parse_args(args)
         cmd = parsed_args.command
@@ -85,7 +116,6 @@ class Destination(Connector, ABC):
         return parsed_args
 
     def run_cmd(self, parsed_args: argparse.Namespace) -> Iterable[AirbyteMessage]:
-
         cmd = parsed_args.command
         if cmd not in self.VALID_CMDS:
             raise Exception(f"Unrecognized command: {cmd}")
@@ -101,7 +131,7 @@ class Destination(Connector, ABC):
             except AirbyteTracedException as traced_exc:
                 connection_status = traced_exc.as_connection_status_message()
                 if connection_status and cmd == "check":
-                    yield connection_status.json(exclude_unset=True)
+                    yield connection_status
                     return
                 raise traced_exc
 
@@ -110,11 +140,15 @@ class Destination(Connector, ABC):
         elif cmd == "write":
             # Wrap in UTF-8 to override any other input encodings
             wrapped_stdin = io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8")
-            yield from self._run_write(config=config, configured_catalog_path=parsed_args.catalog, input_stream=wrapped_stdin)
+            yield from self._run_write(
+                config=config,
+                configured_catalog_path=parsed_args.catalog,
+                input_stream=wrapped_stdin,
+            )
 
-    def run(self, args: List[str]):
+    def run(self, args: List[str]) -> None:
         init_uncaught_exception_handler(logger)
         parsed_args = self.parse_args(args)
         output_messages = self.run_cmd(parsed_args)
         for message in output_messages:
-            print(message.json(exclude_unset=True))
+            print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())

airbyte_cdk/destinations/vector_db_based/README.md

@@ -0,0 +1,37 @@
+# Vector DB based destinations
+
+## Note: All helpers in this directory are experimental and subject to change
+
+This directory contains several helpers that can be used to create a destination that processes and chunks records, embeds their text part and loads them into a vector database.
+The specific loading behavior is defined by the destination connector itself, but chunking and embedding behavior is handled by the helpers.
+
+To use these helpers, install the CDK with the `vector-db-based` extra:
+
+```bash
+pip install airbyte-cdk[vector-db-based]
+```
+
+The helpers can be used in the following way:
+
+- Add the config models to the spec of the connector
+- Implement the `Indexer` interface for your specific database
+- In the check implementation of the destination, initialize the indexer and the embedder and call `check` on them
+- In the write implementation of the destination, initialize the indexer, the embedder and pass them to a new instance of the writer. Then call the writers `write` method with the iterable for incoming messages
+
+If there are no connector-specific embedders, the `airbyte_cdk.destinations.vector_db_based.embedder.create_from_config` function can be used to get an embedder instance from the config.
+
+This is how the components interact:
+
+```text
+┌─────────────┐
+│MyDestination│
+└┬────────────┘
+┌▽───────────────────────────────┐
+│Writer                          │
+└┬─────────┬──────────┬──────────┘
+┌▽───────┐┌▽────────┐┌▽────────────────┐
+│Embedder││MyIndexer││DocumentProcessor│
+└────────┘└─────────┘└─────────────────┘
+```
+
+Normally, only the `MyDestination` class and the `MyIndexer` class has to be implemented specifically for the destination. The other classes are provided as is by the helpers.