airbyte-cdk 0.53.2__tar.gz → 0.53.4__tar.gz

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.53.2
+Version: 0.53.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/airbyte_cdk/destinations/vector_db_based/config.py

@@ -4,6 +4,7 @@
 
 from typing import List, Literal, Optional, Union
 
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
 from pydantic import BaseModel, Field
 
 
@@ -16,11 +17,10 @@ class SeparatorSplitterConfigModel(BaseModel):
     )
     keep_separator: bool = Field(default=False, title="Keep separator", description="Whether to keep the separator in the resulting chunks")
 
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "By Separator"
-        schema_extra = {
-            "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc."
-        }
+        description = "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc."
+        discriminator = "mode"
 
 
 class MarkdownHeaderSplitterConfigModel(BaseModel):
@@ -33,11 +33,10 @@ class MarkdownHeaderSplitterConfigModel(BaseModel):
         ge=1,
     )
 
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "By Markdown header"
-        schema_extra = {
-            "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk."
-        }
+        description = "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk."
+        discriminator = "mode"
 
 
 class CodeSplitterConfigModel(BaseModel):
@@ -65,11 +64,12 @@ class CodeSplitterConfigModel(BaseModel):
         ],
     )
 
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "By Programming Language"
-        schema_extra = {
-            "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks."
-        }
+        description = (
+            "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks."
+        )
+        discriminator = "mode"
 
 
 TextSplitterConfigModel = Union[SeparatorSplitterConfigModel, MarkdownHeaderSplitterConfigModel, CodeSplitterConfigModel]
@@ -128,11 +128,12 @@ class OpenAIEmbeddingConfigModel(BaseModel):
     mode: Literal["openai"] = Field("openai", const=True)
     openai_key: str = Field(..., title="OpenAI API key", airbyte_secret=True)
 
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "OpenAI"
-        schema_extra = {
-            "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions."
-        }
+        description = (
+            "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions."
+        )
+        discriminator = "mode"
 
 
 class OpenAICompatibleEmbeddingConfigModel(BaseModel):
@@ -151,9 +152,10 @@ class OpenAICompatibleEmbeddingConfigModel(BaseModel):
         title="Embedding dimensions", description="The number of dimensions the embedding model is generating", examples=[1536, 384]
     )
 
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "OpenAI-compatible"
-        schema_extra = {"description": "Use a service that's compatible with the OpenAI API to embed text."}
+        description = "Use a service that's compatible with the OpenAI API to embed text."
+        discriminator = "mode"
 
 
 class AzureOpenAIEmbeddingConfigModel(BaseModel):
@@ -177,21 +179,19 @@ class AzureOpenAIEmbeddingConfigModel(BaseModel):
         examples=["your-resource-name"],
     )
 
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "Azure OpenAI"
-        schema_extra = {
-            "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions."
-        }
+        description = "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions."
+        discriminator = "mode"
 
 
 class FakeEmbeddingConfigModel(BaseModel):
     mode: Literal["fake"] = Field("fake", const=True)
 
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "Fake"
-        schema_extra = {
-            "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs."
-        }
+        description = "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs."
+        discriminator = "mode"
 
 
 class FromFieldEmbeddingConfigModel(BaseModel):
@@ -203,17 +203,17 @@ class FromFieldEmbeddingConfigModel(BaseModel):
         ..., title="Embedding dimensions", description="The number of dimensions the embedding model is generating", examples=[1536, 384]
     )
 
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "From Field"
-        schema_extra = {
-            "description": "Use a field in the record as the embedding. This is useful if you already have an embedding for your data and want to store it in the vector store."
-        }
+        description = "Use a field in the record as the embedding. This is useful if you already have an embedding for your data and want to store it in the vector store."
+        discriminator = "mode"
 
 
 class CohereEmbeddingConfigModel(BaseModel):
     mode: Literal["cohere"] = Field("cohere", const=True)
     cohere_key: str = Field(..., title="Cohere API key", airbyte_secret=True)
 
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "Cohere"
-        schema_extra = {"description": "Use the Cohere API to embed text."}
+        description = "Use the Cohere API to embed text."
+        discriminator = "mode"

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/airbyte_cdk/sources/file_based/config/avro_format.py

@@ -2,12 +2,14 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
 from pydantic import BaseModel, Field
 
 
 class AvroFormat(BaseModel):
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "Avro Format"
+        discriminator = "filetype"
 
     filetype: str = Field(
         "avro",

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/airbyte_cdk/sources/file_based/config/csv_format.py

@@ -6,6 +6,7 @@ import codecs
 from enum import Enum
 from typing import Any, Dict, List, Optional, Set, Union
 
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
 from pydantic import BaseModel, Field, ValidationError, root_validator, validator
 
 
@@ -21,8 +22,9 @@ class CsvHeaderDefinitionType(Enum):
 
 
 class CsvHeaderFromCsv(BaseModel):
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "From CSV"
+        discriminator = "header_definition_type"
 
     header_definition_type: str = Field(
         CsvHeaderDefinitionType.FROM_CSV.value,
@@ -34,8 +36,9 @@ class CsvHeaderFromCsv(BaseModel):
 
 
 class CsvHeaderAutogenerated(BaseModel):
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "Autogenerated"
+        discriminator = "header_definition_type"
 
     header_definition_type: str = Field(
         CsvHeaderDefinitionType.AUTOGENERATED.value,
@@ -47,8 +50,9 @@ class CsvHeaderAutogenerated(BaseModel):
 
 
 class CsvHeaderUserProvided(BaseModel):
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "User Provided"
+        discriminator = "header_definition_type"
 
     header_definition_type: str = Field(
         CsvHeaderDefinitionType.USER_PROVIDED.value,
@@ -74,8 +78,9 @@ DEFAULT_FALSE_VALUES = ["n", "no", "f", "false", "off", "0"]
 
 
 class CsvFormat(BaseModel):
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "CSV Format"
+        discriminator = "filetype"
 
     filetype: str = Field(
         "csv",
@@ -123,7 +128,7 @@ class CsvFormat(BaseModel):
     )
     header_definition: Union[CsvHeaderFromCsv, CsvHeaderAutogenerated, CsvHeaderUserProvided] = Field(
         title="CSV Header Definition",
-        default=CsvHeaderFromCsv(),
+        default=CsvHeaderFromCsv(header_definition_type=CsvHeaderDefinitionType.FROM_CSV.value),
         description="How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.",
     )
     true_values: Set[str] = Field(

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/airbyte_cdk/sources/file_based/config/jsonl_format.py

@@ -2,12 +2,14 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
 from pydantic import BaseModel, Field
 
 
 class JsonlFormat(BaseModel):
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "Jsonl Format"
+        discriminator = "filetype"
 
     filetype: str = Field(
         "jsonl",

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/airbyte_cdk/sources/file_based/config/parquet_format.py

@@ -2,12 +2,14 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
 from pydantic import BaseModel, Field
 
 
 class ParquetFormat(BaseModel):
-    class Config:
+    class Config(OneOfOptionConfig):
         title = "Parquet Format"
+        discriminator = "filetype"
 
     filetype: str = Field(
         "parquet",

airbyte-cdk-0.53.4/airbyte_cdk/sources/file_based/config/unstructured_format.py

@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Optional
+
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
+from pydantic import BaseModel, Field
+
+
+class UnstructuredFormat(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Document File Type Format (Experimental)"
+        description = "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file."
+        discriminator = "filetype"
+
+    filetype: str = Field(
+        "unstructured",
+        const=True,
+    )
+
+    skip_unprocessable_file_types: Optional[bool] = Field(
+        default=True,
+        title="Skip Unprocessable File Types",
+        description="If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.",
+        always_show=True,
+    )

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py

@@ -6,6 +6,7 @@ from io import BytesIO, IOBase
 from typing import Any, Dict, Iterable, List, Mapping, Optional
 
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
+from airbyte_cdk.sources.file_based.config.unstructured_format import UnstructuredFormat
 from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
@@ -60,11 +61,12 @@ class UnstructuredParser(FileTypeParser):
         stream_reader: AbstractFileBasedStreamReader,
         logger: logging.Logger,
     ) -> SchemaType:
+        format = _extract_format(config)
         with stream_reader.open_file(file, self.file_read_mode, None, logger) as file_handle:
             filetype = self._get_filetype(file_handle, file)
 
             if filetype not in self._supported_file_types():
-                raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri)
+                self._handle_unprocessable_file(file, format, logger)
 
             return {
                 "content": {"type": "string"},
@@ -79,14 +81,16 @@ class UnstructuredParser(FileTypeParser):
         logger: logging.Logger,
         discovered_schema: Optional[Mapping[str, SchemaType]],
     ) -> Iterable[Dict[str, Any]]:
+        format = _extract_format(config)
         with stream_reader.open_file(file, self.file_read_mode, None, logger) as file_handle:
-            markdown = self._read_file(file_handle, file)
-            yield {
-                "content": markdown,
-                "document_key": file.uri,
-            }
-
-    def _read_file(self, file_handle: IOBase, remote_file: RemoteFile) -> str:
+            markdown = self._read_file(file_handle, file, format, logger)
+            if markdown is not None:
+                yield {
+                    "content": markdown,
+                    "document_key": file.uri,
+                }
+
+    def _read_file(self, file_handle: IOBase, remote_file: RemoteFile, format: UnstructuredFormat, logger: logging.Logger) -> Optional[str]:
         _import_unstructured()
         if (
             (not unstructured_partition_pdf)
@@ -104,7 +108,8 @@ class UnstructuredParser(FileTypeParser):
             decoded_content: str = unstructured_optional_decode(file_content)
             return decoded_content
         if filetype not in self._supported_file_types():
-            raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=remote_file.uri)
+            self._handle_unprocessable_file(remote_file, format, logger)
+            return None
 
         file: Any = file_handle
         if filetype == FileType.PDF:
@@ -120,6 +125,12 @@ class UnstructuredParser(FileTypeParser):
 
         return self._render_markdown(elements)
 
+    def _handle_unprocessable_file(self, remote_file: RemoteFile, format: UnstructuredFormat, logger: logging.Logger) -> None:
+        if format.skip_unprocessable_file_types:
+            logger.warn(f"File {remote_file.uri} cannot be parsed. Skipping it.")
+        else:
+            raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=remote_file.uri)
+
     def _get_filetype(self, file: IOBase, remote_file: RemoteFile) -> Optional[FileType]:
         """
         Detect the file type based on the file name and the file content.
@@ -172,3 +183,10 @@ class UnstructuredParser(FileTypeParser):
     @property
     def file_read_mode(self) -> FileReadMode:
         return FileReadMode.READ_BINARY
+
+
+def _extract_format(config: FileBasedStreamConfig) -> UnstructuredFormat:
+    config_format = config.format
+    if not isinstance(config_format, UnstructuredFormat):
+        raise ValueError(f"Invalid format config: {config_format}")
+    return config_format

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/airbyte_cdk/sources/streams/concurrent/thread_based_concurrent_stream.py

@@ -154,7 +154,7 @@ class ThreadBasedConcurrentStream(AbstractStream):
         if len(futures) < self._max_concurrent_tasks:
             return
 
-        for index in range(len(futures)):
+        for index in reversed(range(len(futures))):
             future = futures[index]
             optional_exception = future.exception()
             if optional_exception:

airbyte-cdk-0.53.4/airbyte_cdk/utils/oneof_option_config.py

@@ -0,0 +1,33 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Any, Dict
+
+
+class OneOfOptionConfig:
+    """
+    Base class to configure a Pydantic model that's used as a oneOf option in a parent model in a way that's compatible with all Airbyte consumers.
+
+    Inherit from this class in the nested Config class in a model and set title and description (these show up in the UI) and discriminator (this is making sure it's marked as required in the schema).
+
+    Usage:
+
+        ```python
+        class OptionModel(BaseModel):
+            mode: Literal["option_a"] = Field("option_a", const=True)
+            option_a_field: str = Field(...)
+
+            class Config(OneOfOptionConfig):
+                title = "Option A"
+                description = "Option A description"
+                discriminator = "mode"
+        ```
+    """
+
+    @staticmethod
+    def schema_extra(schema: Dict[str, Any], model: Any) -> None:
+        if hasattr(model.Config, "description"):
+            schema["description"] = model.Config.description
+        if hasattr(model.Config, "discriminator"):
+            schema.setdefault("required", []).append(model.Config.discriminator)

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.53.2
+Version: 0.53.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/airbyte_cdk.egg-info/SOURCES.txt

@@ -247,6 +247,7 @@ airbyte_cdk/utils/datetime_format_inferrer.py
 airbyte_cdk/utils/event_timing.py
 airbyte_cdk/utils/is_cloud_environment.py
 airbyte_cdk/utils/mapping_helpers.py
+airbyte_cdk/utils/oneof_option_config.py
 airbyte_cdk/utils/schema_inferrer.py
 airbyte_cdk/utils/spec_schema_transformations.py
 airbyte_cdk/utils/stream_status_utils.py

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/setup.py

@@ -36,7 +36,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.53.2",
+    version="0.53.4",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/unit_tests/sources/file_based/file_types/test_unstructured_parser.py

@@ -7,6 +7,7 @@ from datetime import datetime
 from unittest.mock import MagicMock, mock_open, patch
 
 import pytest
+from airbyte_cdk.sources.file_based.config.unstructured_format import UnstructuredFormat
 from airbyte_cdk.sources.file_based.exceptions import RecordParseError
 from airbyte_cdk.sources.file_based.file_types import UnstructuredParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
@@ -17,37 +18,48 @@ FILE_URI = "path/to/file.xyz"
 
 
 @pytest.mark.parametrize(
-    "filetype, raises",
+    "filetype, format_config, raises",
     [
         pytest.param(
             FileType.MD,
+            UnstructuredFormat(skip_unprocessable_file_types=False),
             False,
             id="markdown file",
         ),
         pytest.param(
             FileType.CSV,
+            UnstructuredFormat(skip_unprocessable_file_types=False),
             True,
             id="wrong file format",
         ),
+        pytest.param(
+            FileType.CSV,
+            UnstructuredFormat(skip_unprocessable_file_types=True),
+            False,
+            id="wrong file format skipping",
+        ),
         pytest.param(
             FileType.PDF,
+            UnstructuredFormat(skip_unprocessable_file_types=False),
             False,
             id="pdf file",
         ),
         pytest.param(
             FileType.DOCX,
+            UnstructuredFormat(skip_unprocessable_file_types=False),
             False,
             id="docx file",
         ),
         pytest.param(
             FileType.PPTX,
+            UnstructuredFormat(skip_unprocessable_file_types=False),
             False,
             id="pptx file",
         ),
     ],
 )
 @patch("airbyte_cdk.sources.file_based.file_types.unstructured_parser.detect_filetype")
-def test_infer_schema(mock_detect_filetype, filetype, raises):
+def test_infer_schema(mock_detect_filetype, filetype, format_config, raises):
     # use a fresh event loop to avoid leaking into other tests
     main_loop = asyncio.get_event_loop()
     loop = asyncio.new_event_loop()
@@ -59,11 +71,13 @@ def test_infer_schema(mock_detect_filetype, filetype, raises):
     fake_file.uri = FILE_URI
     logger = MagicMock()
     mock_detect_filetype.return_value = filetype
+    config = MagicMock()
+    config.format = format_config
     if raises:
         with pytest.raises(RecordParseError):
-            loop.run_until_complete(UnstructuredParser().infer_schema(MagicMock(), fake_file, stream_reader, logger))
+            loop.run_until_complete(UnstructuredParser().infer_schema(config, fake_file, stream_reader, logger))
     else:
-        schema = loop.run_until_complete(UnstructuredParser().infer_schema(MagicMock(), MagicMock(), MagicMock(), MagicMock()))
+        schema = loop.run_until_complete(UnstructuredParser().infer_schema(config, MagicMock(), MagicMock(), MagicMock()))
         assert schema == {
             "content": {"type": "string"},
             "document_key": {"type": "string"},
@@ -73,10 +87,11 @@ def test_infer_schema(mock_detect_filetype, filetype, raises):
 
 
 @pytest.mark.parametrize(
-    "filetype, parse_result, raises, expected_records",
+    "filetype, format_config, parse_result, raises, expected_records",
     [
         pytest.param(
             FileType.MD,
+            UnstructuredFormat(skip_unprocessable_file_types=False),
             "test",
             False,
             [
@@ -89,13 +104,23 @@ def test_infer_schema(mock_detect_filetype, filetype, raises):
         ),
         pytest.param(
             FileType.CSV,
-            "test",
+            UnstructuredFormat(skip_unprocessable_file_types=False),
+            None,
             True,
             None,
             id="wrong file format",
         ),
+        pytest.param(
+            FileType.CSV,
+            UnstructuredFormat(skip_unprocessable_file_types=True),
+            None,
+            False,
+            [],
+            id="skip_unprocessable_file_types",
+        ),
         pytest.param(
             FileType.PDF,
+            UnstructuredFormat(skip_unprocessable_file_types=False),
             [
                 Title("heading"),
                 Text("This is the text"),
@@ -113,6 +138,7 @@ def test_infer_schema(mock_detect_filetype, filetype, raises):
         ),
         pytest.param(
             FileType.PDF,
+            UnstructuredFormat(skip_unprocessable_file_types=False),
             [
                 Title("first level heading", metadata=ElementMetadata(category_depth=1)),
                 Title("second level heading", metadata=ElementMetadata(category_depth=2)),
@@ -128,6 +154,7 @@ def test_infer_schema(mock_detect_filetype, filetype, raises):
         ),
         pytest.param(
             FileType.DOCX,
+            UnstructuredFormat(skip_unprocessable_file_types=False),
             [
                 Title("heading"),
                 Text("This is the text"),
@@ -157,6 +184,7 @@ def test_parse_records(
     mock_partition_pptx,
     mock_partition_pdf,
     filetype,
+    format_config,
     parse_result,
     raises,
     expected_records,
@@ -166,6 +194,8 @@ def test_parse_records(
     fake_file = RemoteFile(uri=FILE_URI, last_modified=datetime.now())
     fake_file.uri = FILE_URI
     logger = MagicMock()
+    config = MagicMock()
+    config.format = format_config
     mock_detect_filetype.return_value = filetype
     mock_partition_docx.return_value = parse_result
     mock_partition_pptx.return_value = parse_result
@@ -173,6 +203,6 @@ def test_parse_records(
     mock_optional_decode.side_effect = lambda x: x.decode("utf-8")
     if raises:
         with pytest.raises(RecordParseError):
-            list(UnstructuredParser().parse_records(MagicMock(), fake_file, stream_reader, logger, MagicMock()))
+            list(UnstructuredParser().parse_records(config, fake_file, stream_reader, logger, MagicMock()))
     else:
-        assert list(UnstructuredParser().parse_records(MagicMock(), fake_file, stream_reader, logger, MagicMock())) == expected_records
+        assert list(UnstructuredParser().parse_records(config, fake_file, stream_reader, logger, MagicMock())) == expected_records

{airbyte-cdk-0.53.2 → airbyte-cdk-0.53.4}/unit_tests/sources/file_based/scenarios/csv_scenarios.py

@@ -122,6 +122,7 @@ single_csv_scenario: TestScenario[InMemoryFilesSource] = (
                                                     "type": "boolean",
                                                 },
                                             },
+                                            "required": ["filetype"],
                                         },
                                         {
                                             "title": "CSV Format",
@@ -200,6 +201,7 @@ single_csv_scenario: TestScenario[InMemoryFilesSource] = (
                                                                     "type": "string",
                                                                 },
                                                             },
+                                                            "required": ["header_definition_type"],
                                                         },
                                                         {
                                                             "title": "Autogenerated",
@@ -212,6 +214,7 @@ single_csv_scenario: TestScenario[InMemoryFilesSource] = (
                                                                     "type": "string",
                                                                 },
                                                             },
+                                                            "required": ["header_definition_type"],
                                                         },
                                                         {
                                                             "title": "User Provided",
@@ -230,7 +233,7 @@ single_csv_scenario: TestScenario[InMemoryFilesSource] = (
                                                                     "items": {"type": "string"},
                                                                 },
                                                             },
-                                                            "required": ["column_names"],
+                                                            "required": ["column_names", "header_definition_type"],
                                                         },
                                                     ],
                                                 },
@@ -258,6 +261,7 @@ single_csv_scenario: TestScenario[InMemoryFilesSource] = (
                                                     "enum": ["None", "Primitive Types Only"],
                                                 },
                                             },
+                                            "required": ["filetype"],
                                         },
                                         {
                                             "title": "Jsonl Format",
@@ -265,6 +269,7 @@ single_csv_scenario: TestScenario[InMemoryFilesSource] = (
                                             "properties": {
                                                 "filetype": {"title": "Filetype", "default": "jsonl", "const": "jsonl", "type": "string"}
                                             },
+                                            "required": ["filetype"],
                                         },
                                         {
                                             "title": "Parquet Format",
@@ -283,6 +288,7 @@ single_csv_scenario: TestScenario[InMemoryFilesSource] = (
                                                     "type": "boolean",
                                                 },
                                             },
+                                            "required": ["filetype"],
                                         },
                                         {
                                             "title": "Document File Type Format (Experimental)",
@@ -293,9 +299,17 @@ single_csv_scenario: TestScenario[InMemoryFilesSource] = (
                                                     "default": "unstructured",
                                                     "const": "unstructured",
                                                     "type": "string",
-                                                }
+                                                },
+                                                "skip_unprocessable_file_types": {
+                                                    "type": "boolean",
+                                                    "default": True,
+                                                    "title": "Skip Unprocessable File Types",
+                                                    "description": "If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.",
+                                                    "always_show": True,
+                                                },
                                             },
                                             "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.",
+                                            "required": ["filetype"],
                                         },
                                     ],
                                 },