airbyte-cdk 0.51.10__tar.gz → 0.51.12__tar.gz

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.51.10
+Version: 0.51.12
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/airbyte_cdk/destinations/vector_db_based/config.py

@@ -59,6 +59,22 @@ class FakeEmbeddingConfigModel(BaseModel):
         }
 
 
+class FromFieldEmbeddingConfigModel(BaseModel):
+    mode: Literal["from_field"] = Field("from_field", const=True)
+    field_name: str = Field(
+        ..., title="Field name", description="Name of the field in the record that contains the embedding", examples=["embedding", "vector"]
+    )
+    dimensions: int = Field(
+        ..., title="Embedding dimensions", description="The number of dimensions the embedding model is generating", examples=[1536, 384]
+    )
+
+    class Config:
+        title = "From Field"
+        schema_extra = {
+            "description": "Use a field in the record as the embedding. This is useful if you already have an embedding for your data and want to store it in the vector store."
+        }
+
+
 class CohereEmbeddingConfigModel(BaseModel):
     mode: Literal["cohere"] = Field("cohere", const=True)
     cohere_key: str = Field(..., title="Cohere API key", airbyte_secret=True)

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/airbyte_cdk/destinations/vector_db_based/document_processor.py

@@ -9,6 +9,7 @@ from typing import Any, Dict, List, Mapping, Optional, Tuple, Union
 import dpath.util
 from airbyte_cdk.destinations.vector_db_based.config import ProcessingConfigModel
 from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
 from langchain.document_loaders.base import Document
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.utils import stringify_dict
@@ -21,8 +22,8 @@ METADATA_RECORD_ID_FIELD = "_ab_record_id"
 class Chunk:
     page_content: str
     metadata: Dict[str, Any]
-    stream: str
-    namespace: Optional[str] = None
+    record: AirbyteRecordMessage
+    embedding: Optional[List[float]] = None
 
 
 class DocumentProcessor:
@@ -66,11 +67,14 @@ class DocumentProcessor:
         """
         doc = self._generate_document(record)
         if doc is None:
-            raise ValueError(f"Record {str(record.data)[:250]}... does not contain any text fields.")
-        chunks = [
-            Chunk(
-                page_content=chunk_document.page_content, metadata=chunk_document.metadata, stream=record.stream, namespace=record.namespace
+            text_fields = ", ".join(self.text_fields) if self.text_fields else "all fields"
+            raise AirbyteTracedException(
+                internal_message="No text fields found in record",
+                message=f"Record {str(record.data)[:250]}... does not contain any of the configured text fields: {text_fields}. Please check your processing configuration, there has to be at least one text field set in each record.",
+                failure_type=FailureType.config_error,
             )
+        chunks = [
+            Chunk(page_content=chunk_document.page_content, metadata=chunk_document.metadata, record=record)
             for chunk_document in self._split_document(doc)
         ]
         id_to_delete = doc.metadata[METADATA_RECORD_ID_FIELD] if METADATA_RECORD_ID_FIELD in doc.metadata else None

airbyte-cdk-0.51.12/airbyte_cdk/destinations/vector_db_based/embedder.py

@@ -0,0 +1,164 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+from airbyte_cdk.destinations.vector_db_based.config import (
+    CohereEmbeddingConfigModel,
+    FakeEmbeddingConfigModel,
+    FromFieldEmbeddingConfigModel,
+    OpenAIEmbeddingConfigModel,
+)
+from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk
+from airbyte_cdk.destinations.vector_db_based.utils import format_exception
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType
+from langchain.embeddings.cohere import CohereEmbeddings
+from langchain.embeddings.fake import FakeEmbeddings
+from langchain.embeddings.openai import OpenAIEmbeddings
+
+
+class Embedder(ABC):
+    """
+    Embedder is an abstract class that defines the interface for embedding text.
+
+    The Indexer class uses the Embedder class to internally embed text - each indexer is responsible to pass the text of all documents to the embedder and store the resulting embeddings in the destination.
+    The destination connector is responsible to create an embedder instance and pass it to the writer.
+    The CDK defines basic embedders that should be supported in each destination. It is possible to implement custom embedders for special destinations if needed.
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    @abstractmethod
+    def check(self) -> Optional[str]:
+        pass
+
+    @abstractmethod
+    def embed_chunks(self, chunks: List[Chunk]) -> List[Optional[List[float]]]:
+        """
+        Embed the text of each chunk and return the resulting embedding vectors.
+        If a chunk cannot be embedded or is configured to not be embedded, return None for that chunk.
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def embedding_dimensions(self) -> int:
+        pass
+
+
+OPEN_AI_VECTOR_SIZE = 1536
+
+
+class OpenAIEmbedder(Embedder):
+    def __init__(self, config: OpenAIEmbeddingConfigModel):
+        super().__init__()
+        # Client is set internally
+        self.embeddings = OpenAIEmbeddings(openai_api_key=config.openai_key, chunk_size=8191, max_retries=15)  # type: ignore
+
+    def check(self) -> Optional[str]:
+        try:
+            self.embeddings.embed_query("test")
+        except Exception as e:
+            return format_exception(e)
+        return None
+
+    def embed_chunks(self, chunks: List[Chunk]) -> List[List[float]]:
+        return self.embeddings.embed_documents([chunk.page_content for chunk in chunks])
+
+    @property
+    def embedding_dimensions(self) -> int:
+        # vector size produced by text-embedding-ada-002 model
+        return OPEN_AI_VECTOR_SIZE
+
+
+COHERE_VECTOR_SIZE = 1024
+
+
+class CohereEmbedder(Embedder):
+    def __init__(self, config: CohereEmbeddingConfigModel):
+        super().__init__()
+        # Client is set internally
+        self.embeddings = CohereEmbeddings(cohere_api_key=config.cohere_key, model="embed-english-light-v2.0")  # type: ignore
+
+    def check(self) -> Optional[str]:
+        try:
+            self.embeddings.embed_query("test")
+        except Exception as e:
+            return format_exception(e)
+        return None
+
+    def embed_chunks(self, chunks: List[Chunk]) -> List[List[float]]:
+        return self.embeddings.embed_documents([chunk.page_content for chunk in chunks])
+
+    @property
+    def embedding_dimensions(self) -> int:
+        # vector size produced by text-embedding-ada-002 model
+        return COHERE_VECTOR_SIZE
+
+
+class FakeEmbedder(Embedder):
+    def __init__(self, config: FakeEmbeddingConfigModel):
+        super().__init__()
+        self.embeddings = FakeEmbeddings(size=OPEN_AI_VECTOR_SIZE)
+
+    def check(self) -> Optional[str]:
+        try:
+            self.embeddings.embed_query("test")
+        except Exception as e:
+            return format_exception(e)
+        return None
+
+    def embed_chunks(self, chunks: List[Chunk]) -> List[List[float]]:
+        return self.embeddings.embed_documents([chunk.page_content for chunk in chunks])
+
+    @property
+    def embedding_dimensions(self) -> int:
+        # use same vector size as for OpenAI embeddings to keep it realistic
+        return OPEN_AI_VECTOR_SIZE
+
+
+class FromFieldEmbedder(Embedder):
+    def __init__(self, config: FromFieldEmbeddingConfigModel):
+        super().__init__()
+        self.config = config
+
+    def check(self) -> Optional[str]:
+        return None
+
+    def embed_chunks(self, chunks: List[Chunk]) -> List[List[float]]:
+        """
+        From each chunk, pull the embedding from the field specified in the config.
+        Check that the field exists, is a list of numbers and is the correct size. If not, raise an AirbyteTracedException explaining the problem.
+        """
+        embeddings = []
+        for chunk in chunks:
+            data = chunk.record.data
+            if self.config.field_name not in data:
+                raise AirbyteTracedException(
+                    internal_message="Embedding vector field not found",
+                    failure_type=FailureType.config_error,
+                    message=f"Record {str(data)[:250]}... in stream {chunk.record.stream}  does not contain embedding vector field {self.config.field_name}. Please check your embedding configuration, the embedding vector field has to be set correctly on every record.",
+                )
+            field = data[self.config.field_name]
+            if not isinstance(field, list) or not all(isinstance(x, (int, float)) for x in field):
+                raise AirbyteTracedException(
+                    internal_message="Embedding vector field not a list of numbers",
+                    failure_type=FailureType.config_error,
+                    message=f"Record {str(data)[:250]}...  in stream {chunk.record.stream} does contain embedding vector field {self.config.field_name}, but it is not a list of numbers. Please check your embedding configuration, the embedding vector field has to be a list of numbers of length {self.config.dimensions} on every record.",
+                )
+            if len(field) != self.config.dimensions:
+                raise AirbyteTracedException(
+                    internal_message="Embedding vector field has wrong length",
+                    failure_type=FailureType.config_error,
+                    message=f"Record {str(data)[:250]}...  in stream {chunk.record.stream} does contain embedding vector field {self.config.field_name}, but it has length {len(field)} instead of the configured {self.config.dimensions}. Please check your embedding configuration, the embedding vector field has to be a list of numbers of length {self.config.dimensions} on every record.",
+                )
+            embeddings.append(field)
+
+        return embeddings
+
+    @property
+    def embedding_dimensions(self) -> int:
+        return self.config.dimensions

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/airbyte_cdk/destinations/vector_db_based/indexer.py

@@ -7,7 +7,6 @@ from abc import ABC, abstractmethod
 from typing import Any, Generator, Iterable, List, Optional, Tuple, TypeVar
 
 from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk
-from airbyte_cdk.destinations.vector_db_based.embedder import Embedder
 from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog
 
 
@@ -19,9 +18,8 @@ class Indexer(ABC):
     In a destination connector, implement a custom indexer by extending this class and implementing the abstract methods.
     """
 
-    def __init__(self, config: Any, embedder: Embedder):
+    def __init__(self, config: Any):
         self.config = config
-        self.embedder = embedder
         pass
 
     def pre_sync(self, catalog: ConfiguredAirbyteCatalog) -> None:

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/airbyte_cdk/destinations/vector_db_based/test_utils.py

@@ -48,6 +48,6 @@ class BaseIntegrationTest(unittest.TestCase):
             type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0)
         )
 
-    def setUp(self):
+    def setUp(self) -> None:
         with open("secrets/config.json", "r") as f:
             self.config = json.loads(f.read())

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/airbyte_cdk/destinations/vector_db_based/writer.py

@@ -8,24 +8,27 @@ from typing import Iterable, List
 from airbyte_cdk.destinations.vector_db_based.batcher import Batcher
 from airbyte_cdk.destinations.vector_db_based.config import ProcessingConfigModel
 from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk, DocumentProcessor
+from airbyte_cdk.destinations.vector_db_based.embedder import Embedder
 from airbyte_cdk.destinations.vector_db_based.indexer import Indexer
 from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, ConfiguredAirbyteCatalog, Type
 
 
 class Writer:
     """
-    The Writer class is orchestrating the document processor, the batcher and the indexer:
+    The Writer class is orchestrating the document processor, the batcher, the embedder and the indexer:
     * Incoming records are collected using the batcher
     * The document processor generates documents from all records in the batch
-    * The indexer indexes the resulting documents in the destination
+    * The embedder embeds the documents
+    * The indexer indexes the resulting documents and their embeddings in the destination
 
     The destination connector is responsible to create a writer instance and pass the input messages iterable to the write method.
     The batch size can be configured by the destination connector to give the freedom of either letting the user configure it or hardcoding it to a sensible value depending on the destination.
     """
 
-    def __init__(self, processing_config: ProcessingConfigModel, indexer: Indexer, batch_size: int) -> None:
+    def __init__(self, processing_config: ProcessingConfigModel, indexer: Indexer, embedder: Embedder, batch_size: int) -> None:
         self.processing_config = processing_config
         self.indexer = indexer
+        self.embedder = embedder
         self.batcher = Batcher(batch_size, lambda batch: self._process_batch(batch))
 
     def _process_batch(self, batch: List[AirbyteRecordMessage]) -> None:
@@ -36,6 +39,9 @@ class Writer:
             documents.extend(record_documents)
             if record_id_to_delete is not None:
                 ids_to_delete.append(record_id_to_delete)
+        embeddings = self.embedder.embed_chunks(documents)
+        for i, document in enumerate(documents):
+            document.embedding = embeddings[i]
         self.indexer.index(documents, ids_to_delete)
 
     def write(self, configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage]) -> Iterable[AirbyteMessage]:

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/airbyte_cdk/sources/file_based/config/avro_format.py

@@ -3,14 +3,16 @@
 #
 
 from pydantic import BaseModel, Field
-from typing_extensions import Literal
 
 
 class AvroFormat(BaseModel):
     class Config:
         title = "Avro Format"
 
-    filetype: Literal["avro"] = "avro"
+    filetype: str = Field(
+        "avro",
+        const=True,
+    )
 
     double_as_string: bool = Field(
         title="Convert Double Fields to Strings",

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/airbyte_cdk/sources/file_based/config/csv_format.py

@@ -7,7 +7,6 @@ from enum import Enum
 from typing import Any, Dict, List, Optional, Set, Union
 
 from pydantic import BaseModel, Field, ValidationError, root_validator, validator
-from typing_extensions import Literal
 
 
 class InferenceType(Enum):
@@ -25,7 +24,10 @@ class CsvHeaderFromCsv(BaseModel):
     class Config:
         title = "From CSV"
 
-    header_definition_type: Literal[CsvHeaderDefinitionType.FROM_CSV.value] = CsvHeaderDefinitionType.FROM_CSV.value  # type: ignore
+    header_definition_type: str = Field(
+        CsvHeaderDefinitionType.FROM_CSV.value,
+        const=True,
+    )
 
     def has_header_row(self) -> bool:
         return True
@@ -35,7 +37,10 @@ class CsvHeaderAutogenerated(BaseModel):
     class Config:
         title = "Autogenerated"
 
-    header_definition_type: Literal[CsvHeaderDefinitionType.AUTOGENERATED.value] = CsvHeaderDefinitionType.AUTOGENERATED.value  # type: ignore
+    header_definition_type: str = Field(
+        CsvHeaderDefinitionType.AUTOGENERATED.value,
+        const=True,
+    )
 
     def has_header_row(self) -> bool:
         return False
@@ -45,7 +50,10 @@ class CsvHeaderUserProvided(BaseModel):
     class Config:
         title = "User Provided"
 
-    header_definition_type: Literal[CsvHeaderDefinitionType.USER_PROVIDED.value] = CsvHeaderDefinitionType.USER_PROVIDED.value  # type: ignore
+    header_definition_type: str = Field(
+        CsvHeaderDefinitionType.USER_PROVIDED.value,
+        const=True,
+    )
     column_names: List[str] = Field(
         title="Column Names",
         description="The column names that will be used while emitting the CSV records",
@@ -69,7 +77,10 @@ class CsvFormat(BaseModel):
     class Config:
         title = "CSV Format"
 
-    filetype: Literal["csv"] = "csv"
+    filetype: str = Field(
+        "csv",
+        const=True,
+    )
     delimiter: str = Field(
         title="Delimiter",
         description="The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.",

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/airbyte_cdk/sources/file_based/config/jsonl_format.py

@@ -2,12 +2,14 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from pydantic import BaseModel
-from typing_extensions import Literal
+from pydantic import BaseModel, Field
 
 
 class JsonlFormat(BaseModel):
     class Config:
         title = "Jsonl Format"
 
-    filetype: Literal["jsonl"] = "jsonl"
+    filetype: str = Field(
+        "jsonl",
+        const=True,
+    )

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/airbyte_cdk/sources/file_based/config/parquet_format.py

@@ -3,14 +3,16 @@
 #
 
 from pydantic import BaseModel, Field
-from typing_extensions import Literal
 
 
 class ParquetFormat(BaseModel):
     class Config:
         title = "Parquet Format"
 
-    filetype: Literal["parquet"] = "parquet"
+    filetype: str = Field(
+        "parquet",
+        const=True,
+    )
     # This option is not recommended, but necessary for backwards compatibility
     decimal_as_float: bool = Field(
         title="Convert Decimal Fields to Floats",

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/airbyte_cdk/sources/file_based/file_types/csv_parser.py

@@ -11,6 +11,7 @@ from functools import partial
 from io import IOBase
 from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Set
 
+from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat, CsvHeaderAutogenerated, CsvHeaderUserProvided, InferenceType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError
@@ -18,6 +19,7 @@ from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFile
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.schema_helpers import TYPE_PYTHON_MAPPING, SchemaType
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 DIALECT_NAME = "_config_dialect"
 
@@ -75,11 +77,12 @@ class _CsvReader:
         if isinstance(config_format.header_definition, CsvHeaderUserProvided):
             return config_format.header_definition.column_names  # type: ignore  # should be CsvHeaderUserProvided given the type
 
-        self._skip_rows(fp, config_format.skip_rows_before_header)
         if isinstance(config_format.header_definition, CsvHeaderAutogenerated):
+            self._skip_rows(fp, config_format.skip_rows_before_header + config_format.skip_rows_after_header)
             headers = self._auto_generate_headers(fp, dialect_name)
         else:
             # Then read the header
+            self._skip_rows(fp, config_format.skip_rows_before_header)
             reader = csv.reader(fp, dialect=dialect_name)  # type: ignore
             headers = list(next(reader))
 
@@ -141,6 +144,12 @@ class CsvParser(FileTypeParser):
             if read_bytes >= self._MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE:
                 break
 
+        if not type_inferrer_by_field:
+            raise AirbyteTracedException(
+                message=f"Could not infer schema as there are no rows in {file.uri}. If having an empty CSV file is expected, ignore this. "
+                f"Else, please contact Airbyte.",
+                failure_type=FailureType.config_error,
+            )
         schema = {header.strip(): {"type": type_inferred.infer()} for header, type_inferred in type_inferrer_by_field.items()}
         data_generator.close()
         return schema

airbyte-cdk-0.51.12/airbyte_cdk/sources/file_based/remote_file.py

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from datetime import datetime
+
+from pydantic import BaseModel
+
+
+class RemoteFile(BaseModel):
+    """
+    A file in a file-based stream.
+    """
+
+    uri: str
+    last_modified: datetime

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.51.10
+Version: 0.51.12
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/setup.py

@@ -26,7 +26,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.51.10",
+    version="0.51.12",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/unit_tests/sources/file_based/file_types/test_csv_parser.py

@@ -13,6 +13,7 @@ from unittest import TestCase, mock
 from unittest.mock import Mock
 
 import pytest
+from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.file_based.config.csv_format import (
     DEFAULT_FALSE_VALUES,
     DEFAULT_TRUE_VALUES,
@@ -26,6 +27,7 @@ from airbyte_cdk.sources.file_based.exceptions import RecordParseError
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
 from airbyte_cdk.sources.file_based.file_types.csv_parser import CsvParser, _CsvReader
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 PROPERTY_TYPES = {
     "col1": "null",
@@ -169,7 +171,7 @@ class SchemaInferenceTestCase(TestCase):
         self._config.get_input_schema.return_value = None
         self._config.format = self._config_format
 
-        self._file = Mock(spec=RemoteFile)
+        self._file = RemoteFile(uri="a uri", last_modified=datetime.now())
         self._stream_reader = Mock(spec=AbstractFileBasedStreamReader)
         self._logger = Mock(spec=logging.Logger)
         self._csv_reader = Mock(spec=_CsvReader)
@@ -222,6 +224,12 @@ class SchemaInferenceTestCase(TestCase):
         # since the type is number, we know the string at the end was not considered
         assert inferred_schema == {self._HEADER_NAME: {"type": "number"}}
 
+    def test_given_empty_csv_file_when_infer_schema_then_raise_config_error(self) -> None:
+        self._csv_reader.read_data.return_value = []
+        with pytest.raises(AirbyteTracedException) as exception:
+            self._infer_schema()
+        assert exception.value.failure_type == FailureType.config_error
+
     def _test_infer_schema(self, rows: List[str], expected_type: str) -> None:
         self._csv_reader.read_data.return_value = ({self._HEADER_NAME: row} for row in rows)
         inferred_schema = self._infer_schema()
@@ -260,7 +268,7 @@ class CsvReaderTest(unittest.TestCase):
         self._config.name = self._CONFIG_NAME
         self._config.format = self._config_format
 
-        self._file = Mock(spec=RemoteFile)
+        self._file = RemoteFile(uri="a uri", last_modified=datetime.now())
         self._stream_reader = Mock(spec=AbstractFileBasedStreamReader)
         self._logger = Mock(spec=logging.Logger)
         self._csv_reader = _CsvReader()
@@ -292,6 +300,21 @@ class CsvReaderTest(unittest.TestCase):
 
         assert list(data_generator) == [{"f0": "0", "f1": "1", "f2": "2", "f3": "3", "f4": "4", "f5": "5", "f6": "6"}]
 
+    def test_given_skip_row_before_and_after_and_autogenerated_headers_when_read_data_then_generate_headers_with_format_fX(self) -> None:
+        self._config_format.header_definition = CsvHeaderAutogenerated()
+        self._config_format.skip_rows_before_header = 1
+        self._config_format.skip_rows_after_header = 2
+        self._stream_reader.open_file.return_value = CsvFileBuilder().with_data([
+            "skip before",
+            "skip after 1",
+            "skip after 2",
+            "0,1,2,3,4,5,6"
+        ]).build()
+
+        data_generator = self._read_data()
+
+        assert list(data_generator) == [{"f0": "0", "f1": "1", "f2": "2", "f3": "3", "f4": "4", "f5": "5", "f6": "6"}]
+
     def test_given_user_provided_headers_when_read_data_then_use_user_provided_headers(self) -> None:
         self._config_format.header_definition = CsvHeaderUserProvided(column_names=["first", "second", "third", "fourth"])
         self._stream_reader.open_file.return_value = CsvFileBuilder().with_data(["0,1,2,3"]).build()

{airbyte-cdk-0.51.10 → airbyte-cdk-0.51.12}/unit_tests/sources/file_based/scenarios/csv_scenarios.py

@@ -111,7 +111,7 @@ single_csv_scenario = (
                                             "title": "Avro Format",
                                             "type": "object",
                                             "properties": {
-                                                "filetype": {"title": "Filetype", "default": "avro", "enum": ["avro"], "type": "string"},
+                                                "filetype": {"title": "Filetype", "default": "avro", "const": "avro", "type": "string"},
                                                 "double_as_string": {
                                                     "title": "Convert Double Fields to Strings",
                                                     "description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.",
@@ -124,7 +124,7 @@ single_csv_scenario = (
                                             "title": "CSV Format",
                                             "type": "object",
                                             "properties": {
-                                                "filetype": {"title": "Filetype", "default": "csv", "enum": ["csv"], "type": "string"},
+                                                "filetype": {"title": "Filetype", "default": "csv", "const": "csv", "type": "string"},
                                                 "delimiter": {
                                                     "title": "Delimiter",
                                                     "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.",
@@ -190,21 +190,21 @@ single_csv_scenario = (
                                                             "title": "From CSV",
                                                             "type": "object",
                                                             "properties": {
-                                                                "header_definition_type": {"title": "Header Definition Type", "default": "From CSV", "enum": ["From CSV"], "type": "string"},
+                                                                "header_definition_type": {"title": "Header Definition Type", "default": "From CSV", "const": "From CSV", "type": "string"},
                                                             },
                                                         },
                                                         {
                                                             "title": "Autogenerated",
                                                             "type": "object",
                                                             "properties": {
-                                                                "header_definition_type": {"title": "Header Definition Type", "default": "Autogenerated", "enum": ["Autogenerated"], "type": "string"},
+                                                                "header_definition_type": {"title": "Header Definition Type", "default": "Autogenerated", "const": "Autogenerated", "type": "string"},
                                                             },
                                                         },
                                                         {
                                                             "title": "User Provided",
                                                             "type": "object",
                                                             "properties": {
-                                                                "header_definition_type": {"title": "Header Definition Type", "default": "User Provided", "enum": ["User Provided"], "type": "string"},
+                                                                "header_definition_type": {"title": "Header Definition Type", "default": "User Provided", "const": "User Provided", "type": "string"},
                                                                 "column_names": {
                                                                     "title": "Column Names",
                                                                     "description": "The column names that will be used while emitting the CSV records",
@@ -247,7 +247,7 @@ single_csv_scenario = (
                                             "title": "Jsonl Format",
                                             "type": "object",
                                             "properties": {
-                                                "filetype": {"title": "Filetype", "default": "jsonl", "enum": ["jsonl"], "type": "string"}
+                                                "filetype": {"title": "Filetype", "default": "jsonl", "const": "jsonl", "type": "string"}
                                             },
                                         },
                                         {
@@ -257,7 +257,7 @@ single_csv_scenario = (
                                                 "filetype": {
                                                     "title": "Filetype",
                                                     "default": "parquet",
-                                                    "enum": ["parquet"],
+                                                    "const": "parquet",
                                                     "type": "string",
                                                 },
                                                 "decimal_as_float": {