airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/destinations/vector_db_based/document_processor.py

@@ -0,0 +1,223 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import json
+import logging
+from dataclasses import dataclass
+from typing import Any, Dict, List, Mapping, Optional, Tuple
+
+import dpath
+from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
+from langchain.utils import stringify_dict
+from langchain_core.documents.base import Document
+
+from airbyte_cdk.destinations.vector_db_based.config import (
+    ProcessingConfigModel,
+    SeparatorSplitterConfigModel,
+    TextSplitterConfigModel,
+)
+from airbyte_cdk.destinations.vector_db_based.utils import create_stream_identifier
+from airbyte_cdk.models import (
+    AirbyteRecordMessage,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    DestinationSyncMode,
+)
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
+
+METADATA_STREAM_FIELD = "_ab_stream"
+METADATA_RECORD_ID_FIELD = "_ab_record_id"
+
+CDC_DELETED_FIELD = "_ab_cdc_deleted_at"
+
+
+@dataclass
+class Chunk:
+    page_content: Optional[str]
+    metadata: Dict[str, Any]
+    record: AirbyteRecordMessage
+    embedding: Optional[List[float]] = None
+
+
+headers_to_split_on = [
+    "(?:^|\n)# ",
+    "(?:^|\n)## ",
+    "(?:^|\n)### ",
+    "(?:^|\n)#### ",
+    "(?:^|\n)##### ",
+    "(?:^|\n)###### ",
+]
+
+
+class DocumentProcessor:
+    """
+    DocumentProcessor is a helper class that generates documents from Airbyte records.
+
+    It is used to generate documents from records before writing them to the destination:
+    * The text fields are extracted from the record and concatenated to a single string.
+    * The metadata fields are extracted from the record and added to the document metadata.
+    * The document is split into chunks of a given size using a langchain text splitter.
+
+    The Writer class uses the DocumentProcessor class to internally generate documents from records - in most cases you don't need to use it directly,
+    except if you want to implement a custom writer.
+
+    The config parameters specified by the ProcessingConfigModel has to be made part of the connector spec to allow the user to configure the document processor.
+    Calling DocumentProcessor.check_config(config) will validate the config and return an error message if the config is invalid.
+    """
+
+    streams: Mapping[str, ConfiguredAirbyteStream]
+
+    @staticmethod
+    def check_config(config: ProcessingConfigModel) -> Optional[str]:
+        if config.text_splitter is not None and config.text_splitter.mode == "separator":
+            for s in config.text_splitter.separators:
+                try:
+                    separator = json.loads(s)
+                    if not isinstance(separator, str):
+                        return f"Invalid separator: {s}. Separator needs to be a valid JSON string using double quotes."
+                except json.decoder.JSONDecodeError:
+                    return f"Invalid separator: {s}. Separator needs to be a valid JSON string using double quotes."
+        return None
+
+    def _get_text_splitter(
+        self,
+        chunk_size: int,
+        chunk_overlap: int,
+        splitter_config: Optional[TextSplitterConfigModel],
+    ) -> RecursiveCharacterTextSplitter:
+        if splitter_config is None:
+            splitter_config = SeparatorSplitterConfigModel(mode="separator")
+        if splitter_config.mode == "separator":
+            return RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                separators=[json.loads(s) for s in splitter_config.separators],
+                keep_separator=splitter_config.keep_separator,
+                disallowed_special=(),
+            )
+        if splitter_config.mode == "markdown":
+            return RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                separators=headers_to_split_on[: splitter_config.split_level],
+                is_separator_regex=True,
+                keep_separator=True,
+                disallowed_special=(),
+            )
+        if splitter_config.mode == "code":
+            return RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                separators=RecursiveCharacterTextSplitter.get_separators_for_language(
+                    Language(splitter_config.language)
+                ),
+                disallowed_special=(),
+            )
+
+    def __init__(self, config: ProcessingConfigModel, catalog: ConfiguredAirbyteCatalog):
+        self.streams = {
+            create_stream_identifier(stream.stream): stream for stream in catalog.streams
+        }
+
+        self.splitter = self._get_text_splitter(
+            config.chunk_size, config.chunk_overlap, config.text_splitter
+        )
+        self.text_fields = config.text_fields
+        self.metadata_fields = config.metadata_fields
+        self.field_name_mappings = config.field_name_mappings
+        self.logger = logging.getLogger("airbyte.document_processor")
+
+    def process(self, record: AirbyteRecordMessage) -> Tuple[List[Chunk], Optional[str]]:
+        """
+        Generate documents from records.
+        :param records: List of AirbyteRecordMessages
+        :return: Tuple of (List of document chunks, record id to delete if a stream is in dedup mode to avoid stale documents in the vector store)
+        """
+        if CDC_DELETED_FIELD in record.data and record.data[CDC_DELETED_FIELD]:
+            return [], self._extract_primary_key(record)
+        doc = self._generate_document(record)
+        if doc is None:
+            text_fields = ", ".join(self.text_fields) if self.text_fields else "all fields"
+            raise AirbyteTracedException(
+                internal_message="No text fields found in record",
+                message=f"Record {str(record.data)[:250]}... does not contain any of the configured text fields: {text_fields}. Please check your processing configuration, there has to be at least one text field set in each record.",
+                failure_type=FailureType.config_error,
+            )
+        chunks = [
+            Chunk(
+                page_content=chunk_document.page_content,
+                metadata=chunk_document.metadata,
+                record=record,
+            )
+            for chunk_document in self._split_document(doc)
+        ]
+        id_to_delete = (
+            doc.metadata[METADATA_RECORD_ID_FIELD]
+            if METADATA_RECORD_ID_FIELD in doc.metadata
+            else None
+        )
+        return chunks, id_to_delete
+
+    def _generate_document(self, record: AirbyteRecordMessage) -> Optional[Document]:
+        relevant_fields = self._extract_relevant_fields(record, self.text_fields)
+        if len(relevant_fields) == 0:
+            return None
+        text = stringify_dict(relevant_fields)
+        metadata = self._extract_metadata(record)
+        return Document(page_content=text, metadata=metadata)
+
+    def _extract_relevant_fields(
+        self, record: AirbyteRecordMessage, fields: Optional[List[str]]
+    ) -> Dict[str, Any]:
+        relevant_fields = {}
+        if fields and len(fields) > 0:
+            for field in fields:
+                values = dpath.values(record.data, field, separator=".")
+                if values and len(values) > 0:
+                    relevant_fields[field] = values if len(values) > 1 else values[0]
+        else:
+            relevant_fields = record.data
+        return self._remap_field_names(relevant_fields)
+
+    def _extract_metadata(self, record: AirbyteRecordMessage) -> Dict[str, Any]:
+        metadata = self._extract_relevant_fields(record, self.metadata_fields)
+        metadata[METADATA_STREAM_FIELD] = create_stream_identifier(record)
+        primary_key = self._extract_primary_key(record)
+        if primary_key:
+            metadata[METADATA_RECORD_ID_FIELD] = primary_key
+        return metadata
+
+    def _extract_primary_key(self, record: AirbyteRecordMessage) -> Optional[str]:
+        stream_identifier = create_stream_identifier(record)
+        current_stream: ConfiguredAirbyteStream = self.streams[stream_identifier]
+        # if the sync mode is deduping, use the primary key to upsert existing records instead of appending new ones
+        if (
+            not current_stream.primary_key
+            or current_stream.destination_sync_mode != DestinationSyncMode.append_dedup
+        ):
+            return None
+
+        primary_key = []
+        for key in current_stream.primary_key:
+            try:
+                primary_key.append(str(dpath.get(record.data, key)))
+            except KeyError:
+                primary_key.append("__not_found__")
+        stringified_primary_key = "_".join(primary_key)
+        return f"{stream_identifier}_{stringified_primary_key}"
+
+    def _split_document(self, doc: Document) -> List[Document]:
+        chunks: List[Document] = self.splitter.split_documents([doc])
+        return chunks
+
+    def _remap_field_names(self, fields: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.field_name_mappings:
+            return fields
+
+        new_fields = fields.copy()
+        for mapping in self.field_name_mappings:
+            if mapping.from_field in new_fields:
+                new_fields[mapping.to_field] = new_fields.pop(mapping.from_field)
+
+        return new_fields

airbyte_cdk/destinations/vector_db_based/embedder.py

@@ -0,0 +1,303 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import os
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import List, Optional, Union, cast
+
+from langchain.embeddings.cohere import CohereEmbeddings
+from langchain.embeddings.fake import FakeEmbeddings
+from langchain.embeddings.localai import LocalAIEmbeddings
+from langchain.embeddings.openai import OpenAIEmbeddings
+
+from airbyte_cdk.destinations.vector_db_based.config import (
+    AzureOpenAIEmbeddingConfigModel,
+    CohereEmbeddingConfigModel,
+    FakeEmbeddingConfigModel,
+    FromFieldEmbeddingConfigModel,
+    OpenAICompatibleEmbeddingConfigModel,
+    OpenAIEmbeddingConfigModel,
+    ProcessingConfigModel,
+)
+from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, format_exception
+from airbyte_cdk.models import AirbyteRecordMessage
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
+
+
+@dataclass
+class Document:
+    page_content: str
+    record: AirbyteRecordMessage
+
+
+class Embedder(ABC):
+    """
+    Embedder is an abstract class that defines the interface for embedding text.
+
+    The Indexer class uses the Embedder class to internally embed text - each indexer is responsible to pass the text of all documents to the embedder and store the resulting embeddings in the destination.
+    The destination connector is responsible to create an embedder instance and pass it to the writer.
+    The CDK defines basic embedders that should be supported in each destination. It is possible to implement custom embedders for special destinations if needed.
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    @abstractmethod
+    def check(self) -> Optional[str]:
+        pass
+
+    @abstractmethod
+    def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]:
+        """
+        Embed the text of each chunk and return the resulting embedding vectors.
+        If a chunk cannot be embedded or is configured to not be embedded, return None for that chunk.
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def embedding_dimensions(self) -> int:
+        pass
+
+
+OPEN_AI_VECTOR_SIZE = 1536
+
+OPEN_AI_TOKEN_LIMIT = 150_000  # limit of tokens per minute
+
+
+class BaseOpenAIEmbedder(Embedder):
+    def __init__(self, embeddings: OpenAIEmbeddings, chunk_size: int):
+        super().__init__()
+        self.embeddings = embeddings
+        self.chunk_size = chunk_size
+
+    def check(self) -> Optional[str]:
+        try:
+            self.embeddings.embed_query("test")
+        except Exception as e:
+            return format_exception(e)
+        return None
+
+    def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]:
+        """
+        Embed the text of each chunk and return the resulting embedding vectors.
+
+        As the OpenAI API will fail if more than the per-minute limit worth of tokens is sent at once, we split the request into batches and embed each batch separately.
+        It's still possible to run into the rate limit between each embed call because the available token budget hasn't recovered between the calls,
+        but the built-in retry mechanism of the OpenAI client handles that.
+        """
+        # Each chunk can hold at most self.chunk_size tokens, so tokens-per-minute by maximum tokens per chunk is the number of documents that can be embedded at once without exhausting the limit in a single request
+        embedding_batch_size = OPEN_AI_TOKEN_LIMIT // self.chunk_size
+        batches = create_chunks(documents, batch_size=embedding_batch_size)
+        embeddings: List[Optional[List[float]]] = []
+        for batch in batches:
+            embeddings.extend(
+                self.embeddings.embed_documents([chunk.page_content for chunk in batch])
+            )
+        return embeddings
+
+    @property
+    def embedding_dimensions(self) -> int:
+        # vector size produced by text-embedding-ada-002 model
+        return OPEN_AI_VECTOR_SIZE
+
+
+class OpenAIEmbedder(BaseOpenAIEmbedder):
+    def __init__(self, config: OpenAIEmbeddingConfigModel, chunk_size: int):
+        super().__init__(
+            OpenAIEmbeddings(  # type: ignore [call-arg]
+                openai_api_key=config.openai_key, max_retries=15, disallowed_special=()
+            ),
+            chunk_size,
+        )  # type: ignore
+
+
+class AzureOpenAIEmbedder(BaseOpenAIEmbedder):
+    def __init__(self, config: AzureOpenAIEmbeddingConfigModel, chunk_size: int):
+        # Azure OpenAI API has — as of 20230927 — a limit of 16 documents per request
+        super().__init__(
+            OpenAIEmbeddings(  # type: ignore [call-arg]
+                openai_api_key=config.openai_key,
+                chunk_size=16,
+                max_retries=15,
+                openai_api_type="azure",
+                openai_api_version="2023-05-15",
+                openai_api_base=config.api_base,
+                deployment=config.deployment,
+                disallowed_special=(),
+            ),
+            chunk_size,
+        )  # type: ignore
+
+
+COHERE_VECTOR_SIZE = 1024
+
+
+class CohereEmbedder(Embedder):
+    def __init__(self, config: CohereEmbeddingConfigModel):
+        super().__init__()
+        # Client is set internally
+        self.embeddings = CohereEmbeddings(
+            cohere_api_key=config.cohere_key, model="embed-english-light-v2.0"
+        )  # type: ignore
+
+    def check(self) -> Optional[str]:
+        try:
+            self.embeddings.embed_query("test")
+        except Exception as e:
+            return format_exception(e)
+        return None
+
+    def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]:
+        return cast(
+            List[Optional[List[float]]],
+            self.embeddings.embed_documents([document.page_content for document in documents]),
+        )
+
+    @property
+    def embedding_dimensions(self) -> int:
+        # vector size produced by text-embedding-ada-002 model
+        return COHERE_VECTOR_SIZE
+
+
+class FakeEmbedder(Embedder):
+    def __init__(self, config: FakeEmbeddingConfigModel):
+        super().__init__()
+        self.embeddings = FakeEmbeddings(size=OPEN_AI_VECTOR_SIZE)
+
+    def check(self) -> Optional[str]:
+        try:
+            self.embeddings.embed_query("test")
+        except Exception as e:
+            return format_exception(e)
+        return None
+
+    def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]:
+        return cast(
+            List[Optional[List[float]]],
+            self.embeddings.embed_documents([document.page_content for document in documents]),
+        )
+
+    @property
+    def embedding_dimensions(self) -> int:
+        # use same vector size as for OpenAI embeddings to keep it realistic
+        return OPEN_AI_VECTOR_SIZE
+
+
+CLOUD_DEPLOYMENT_MODE = "cloud"
+
+
+class OpenAICompatibleEmbedder(Embedder):
+    def __init__(self, config: OpenAICompatibleEmbeddingConfigModel):
+        super().__init__()
+        self.config = config
+        # Client is set internally
+        # Always set an API key even if there is none defined in the config because the validator will fail otherwise. Embedding APIs that don't require an API key don't fail if one is provided, so this is not breaking usage.
+        self.embeddings = LocalAIEmbeddings(
+            model=config.model_name,
+            openai_api_key=config.api_key or "dummy-api-key",
+            openai_api_base=config.base_url,
+            max_retries=15,
+            disallowed_special=(),
+        )  # type: ignore
+
+    def check(self) -> Optional[str]:
+        deployment_mode = os.environ.get("DEPLOYMENT_MODE", "")
+        if (
+            deployment_mode.casefold() == CLOUD_DEPLOYMENT_MODE
+            and not self.config.base_url.startswith("https://")
+        ):
+            return "Base URL must start with https://"
+
+        try:
+            self.embeddings.embed_query("test")
+        except Exception as e:
+            return format_exception(e)
+        return None
+
+    def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]:
+        return cast(
+            List[Optional[List[float]]],
+            self.embeddings.embed_documents([document.page_content for document in documents]),
+        )
+
+    @property
+    def embedding_dimensions(self) -> int:
+        # vector size produced by the model
+        return self.config.dimensions
+
+
+class FromFieldEmbedder(Embedder):
+    def __init__(self, config: FromFieldEmbeddingConfigModel):
+        super().__init__()
+        self.config = config
+
+    def check(self) -> Optional[str]:
+        return None
+
+    def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]:
+        """
+        From each chunk, pull the embedding from the field specified in the config.
+        Check that the field exists, is a list of numbers and is the correct size. If not, raise an AirbyteTracedException explaining the problem.
+        """
+        embeddings: List[Optional[List[float]]] = []
+        for document in documents:
+            data = document.record.data
+            if self.config.field_name not in data:
+                raise AirbyteTracedException(
+                    internal_message="Embedding vector field not found",
+                    failure_type=FailureType.config_error,
+                    message=f"Record {str(data)[:250]}... in stream {document.record.stream}  does not contain embedding vector field {self.config.field_name}. Please check your embedding configuration, the embedding vector field has to be set correctly on every record.",
+                )
+            field = data[self.config.field_name]
+            if not isinstance(field, list) or not all(isinstance(x, (int, float)) for x in field):
+                raise AirbyteTracedException(
+                    internal_message="Embedding vector field not a list of numbers",
+                    failure_type=FailureType.config_error,
+                    message=f"Record {str(data)[:250]}...  in stream {document.record.stream} does contain embedding vector field {self.config.field_name}, but it is not a list of numbers. Please check your embedding configuration, the embedding vector field has to be a list of numbers of length {self.config.dimensions} on every record.",
+                )
+            if len(field) != self.config.dimensions:
+                raise AirbyteTracedException(
+                    internal_message="Embedding vector field has wrong length",
+                    failure_type=FailureType.config_error,
+                    message=f"Record {str(data)[:250]}...  in stream {document.record.stream} does contain embedding vector field {self.config.field_name}, but it has length {len(field)} instead of the configured {self.config.dimensions}. Please check your embedding configuration, the embedding vector field has to be a list of numbers of length {self.config.dimensions} on every record.",
+                )
+            embeddings.append(field)
+
+        return embeddings
+
+    @property
+    def embedding_dimensions(self) -> int:
+        return self.config.dimensions
+
+
+embedder_map = {
+    "openai": OpenAIEmbedder,
+    "cohere": CohereEmbedder,
+    "fake": FakeEmbedder,
+    "azure_openai": AzureOpenAIEmbedder,
+    "from_field": FromFieldEmbedder,
+    "openai_compatible": OpenAICompatibleEmbedder,
+}
+
+
+def create_from_config(
+    embedding_config: Union[
+        AzureOpenAIEmbeddingConfigModel,
+        CohereEmbeddingConfigModel,
+        FakeEmbeddingConfigModel,
+        FromFieldEmbeddingConfigModel,
+        OpenAIEmbeddingConfigModel,
+        OpenAICompatibleEmbeddingConfigModel,
+    ],
+    processing_config: ProcessingConfigModel,
+) -> Embedder:
+    if embedding_config.mode == "azure_openai" or embedding_config.mode == "openai":
+        return cast(
+            Embedder,
+            embedder_map[embedding_config.mode](embedding_config, processing_config.chunk_size),
+        )
+    else:
+        return cast(Embedder, embedder_map[embedding_config.mode](embedding_config))

airbyte_cdk/destinations/vector_db_based/indexer.py

@@ -0,0 +1,78 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import itertools
+from abc import ABC, abstractmethod
+from typing import Any, Generator, Iterable, List, Optional, Tuple, TypeVar
+
+from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk
+from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog
+
+
+class Indexer(ABC):
+    """
+    Indexer is an abstract class that defines the interface for indexing documents.
+
+    The Writer class uses the Indexer class to internally index documents generated by the document processor.
+    In a destination connector, implement a custom indexer by extending this class and implementing the abstract methods.
+    """
+
+    def __init__(self, config: Any):
+        self.config = config
+        pass
+
+    def pre_sync(self, catalog: ConfiguredAirbyteCatalog) -> None:
+        """
+        Run before the sync starts. This method should be used to make sure all records in the destination that belong to streams with a destination mode of overwrite are deleted.
+
+        Each record has a metadata field with the name airbyte_cdk.destinations.vector_db_based.document_processor.METADATA_STREAM_FIELD which can be used to filter documents for deletion.
+        Use the airbyte_cdk.destinations.vector_db_based.utils.create_stream_identifier method to create the stream identifier based on the stream definition to use for filtering.
+        """
+        pass
+
+    def post_sync(self) -> List[AirbyteMessage]:
+        """
+        Run after the sync finishes. This method should be used to perform any cleanup operations and can return a list of AirbyteMessages to be logged.
+        """
+        return []
+
+    @abstractmethod
+    def index(self, document_chunks: List[Chunk], namespace: str, stream: str) -> None:
+        """
+        Index a list of document chunks.
+
+        This method should be used to index the documents in the destination. If page_content is None, the document should be indexed without the raw text.
+        All chunks belong to the stream and namespace specified in the parameters.
+        """
+        pass
+
+    @abstractmethod
+    def delete(self, delete_ids: List[str], namespace: str, stream: str) -> None:
+        """
+        Delete document chunks belonging to certain record ids.
+
+        This method should be used to delete documents from the destination.
+        The delete_ids parameter contains a list of record ids - all chunks with a record id in this list should be deleted from the destination.
+        All ids belong to the stream and namespace specified in the parameters.
+        """
+        pass
+
+    @abstractmethod
+    def check(self) -> Optional[str]:
+        """
+        Check if the indexer is configured correctly. This method should be used to check if the indexer is configured correctly and return an error message if it is not.
+        """
+        pass
+
+
+T = TypeVar("T")
+
+
+def chunks(iterable: Iterable[T], batch_size: int) -> Generator[Tuple[T, ...], None, None]:
+    """A helper function to break an iterable into chunks of size batch_size."""
+    it = iter(iterable)
+    chunk = tuple(itertools.islice(it, batch_size))
+    while chunk:
+        yield chunk
+        chunk = tuple(itertools.islice(it, batch_size))

airbyte_cdk/destinations/vector_db_based/test_utils.py

@@ -0,0 +1,63 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import json
+import unittest
+from typing import Any, Dict
+
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    AirbyteStateMessage,
+    AirbyteStream,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    DestinationSyncMode,
+    SyncMode,
+    Type,
+)
+
+
+class BaseIntegrationTest(unittest.TestCase):
+    """
+    BaseIntegrationTest is a base class for integration tests for vector db destinations.
+
+    It provides helper methods to create Airbyte catalogs, records and state messages.
+    """
+
+    def _get_configured_catalog(
+        self, destination_mode: DestinationSyncMode
+    ) -> ConfiguredAirbyteCatalog:
+        stream_schema = {
+            "type": "object",
+            "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}},
+        }
+
+        overwrite_stream = ConfiguredAirbyteStream(
+            stream=AirbyteStream(
+                name="mystream",
+                json_schema=stream_schema,
+                supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh],
+            ),
+            primary_key=[["int_col"]],
+            sync_mode=SyncMode.incremental,
+            destination_sync_mode=destination_mode,
+        )
+
+        return ConfiguredAirbyteCatalog(streams=[overwrite_stream])
+
+    def _state(self, data: Dict[str, Any]) -> AirbyteMessage:
+        return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data))
+
+    def _record(self, stream: str, str_value: str, int_value: int) -> AirbyteMessage:
+        return AirbyteMessage(
+            type=Type.RECORD,
+            record=AirbyteRecordMessage(
+                stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0
+            ),
+        )
+
+    def setUp(self) -> None:
+        with open("secrets/config.json", "r") as f:
+            self.config = json.loads(f.read())