airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

airbyte_cdk/sources/file_based/__init__.py

@@ -0,0 +1,24 @@
+from .config.abstract_file_based_spec import AbstractFileBasedSpec
+from .config.csv_format import CsvFormat
+from .config.file_based_stream_config import FileBasedStreamConfig
+from .config.jsonl_format import JsonlFormat
+from .exceptions import CustomFileBasedException, ErrorListingFiles, FileBasedSourceError
+from .file_based_source import DEFAULT_CONCURRENCY, FileBasedSource
+from .file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
+from .remote_file import RemoteFile
+from .stream.cursor import DefaultFileBasedCursor
+
+__all__ = [
+    "AbstractFileBasedSpec",
+    "AbstractFileBasedStreamReader",
+    "CsvFormat",
+    "CustomFileBasedException",
+    "DefaultFileBasedCursor",
+    "ErrorListingFiles",
+    "FileBasedSource",
+    "FileBasedSourceError",
+    "FileBasedStreamConfig",
+    "FileReadMode",
+    "JsonlFormat",
+    "RemoteFile",
+]

airbyte_cdk/sources/file_based/availability_strategy/__init__.py

@@ -1,4 +1,11 @@
-from .abstract_file_based_availability_strategy import AbstractFileBasedAvailabilityStrategy, AbstractFileBasedAvailabilityStrategyWrapper
+from .abstract_file_based_availability_strategy import (
+    AbstractFileBasedAvailabilityStrategy,
+    AbstractFileBasedAvailabilityStrategyWrapper,
+)
 from .default_file_based_availability_strategy import DefaultFileBasedAvailabilityStrategy
 
-__all__ = ["AbstractFileBasedAvailabilityStrategy", "AbstractFileBasedAvailabilityStrategyWrapper", "DefaultFileBasedAvailabilityStrategy"]
+__all__ = [
+    "AbstractFileBasedAvailabilityStrategy",
+    "AbstractFileBasedAvailabilityStrategyWrapper",
+    "DefaultFileBasedAvailabilityStrategy",
+]

airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py

@@ -2,6 +2,8 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+from __future__ import annotations
+
 import logging
 from abc import abstractmethod
 from typing import TYPE_CHECKING, Optional, Tuple
@@ -22,7 +24,12 @@ if TYPE_CHECKING:
 
 class AbstractFileBasedAvailabilityStrategy(AvailabilityStrategy):
     @abstractmethod
-    def check_availability(self, stream: Stream, logger: logging.Logger, _: Optional[Source]) -> Tuple[bool, Optional[str]]:
+    def check_availability(  # type: ignore[override]  # Signature doesn't match base class
+        self,
+        stream: Stream,
+        logger: logging.Logger,
+        _: Optional[Source],
+    ) -> Tuple[bool, Optional[str]]:
         """
         Perform a connection check for the stream.
 
@@ -32,7 +39,10 @@ class AbstractFileBasedAvailabilityStrategy(AvailabilityStrategy):
 
     @abstractmethod
     def check_availability_and_parsability(
-        self, stream: "AbstractFileBasedStream", logger: logging.Logger, _: Optional[Source]
+        self,
+        stream: AbstractFileBasedStream,
+        logger: logging.Logger,
+        _: Optional[Source],
     ) -> Tuple[bool, Optional[str]]:
         """
         Performs a connection check for the stream, as well as additional checks that
@@ -44,14 +54,20 @@ class AbstractFileBasedAvailabilityStrategy(AvailabilityStrategy):
 
 
 class AbstractFileBasedAvailabilityStrategyWrapper(AbstractAvailabilityStrategy):
-    def __init__(self, stream: "AbstractFileBasedStream"):
+    def __init__(self, stream: AbstractFileBasedStream) -> None:
         self.stream = stream
 
     def check_availability(self, logger: logging.Logger) -> StreamAvailability:
-        is_available, reason = self.stream.availability_strategy.check_availability(self.stream, logger, None)
+        is_available, reason = self.stream.availability_strategy.check_availability(
+            self.stream, logger, None
+        )
         if is_available:
             return StreamAvailable()
         return StreamUnavailable(reason or "")
 
-    def check_availability_and_parsability(self, logger: logging.Logger) -> Tuple[bool, Optional[str]]:
-        return self.stream.availability_strategy.check_availability_and_parsability(self.stream, logger, None)
+    def check_availability_and_parsability(
+        self, logger: logging.Logger
+    ) -> Tuple[bool, Optional[str]]:
+        return self.stream.availability_strategy.check_availability_and_parsability(
+            self.stream, logger, None
+        )

airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py

@@ -2,13 +2,22 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+from __future__ import annotations
+
 import logging
 import traceback
 from typing import TYPE_CHECKING, Optional, Tuple
 
+from airbyte_cdk import AirbyteTracedException
 from airbyte_cdk.sources import Source
-from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy
-from airbyte_cdk.sources.file_based.exceptions import CheckAvailabilityError, CustomFileBasedException, FileBasedSourceError
+from airbyte_cdk.sources.file_based.availability_strategy import (
+    AbstractFileBasedAvailabilityStrategy,
+)
+from airbyte_cdk.sources.file_based.exceptions import (
+    CheckAvailabilityError,
+    CustomFileBasedException,
+    FileBasedSourceError,
+)
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.schema_helpers import conforms_to_schema
@@ -18,10 +27,15 @@ if TYPE_CHECKING:
 
 
 class DefaultFileBasedAvailabilityStrategy(AbstractFileBasedAvailabilityStrategy):
-    def __init__(self, stream_reader: AbstractFileBasedStreamReader):
+    def __init__(self, stream_reader: AbstractFileBasedStreamReader) -> None:
         self.stream_reader = stream_reader
 
-    def check_availability(self, stream: "AbstractFileBasedStream", logger: logging.Logger, _: Optional[Source]) -> Tuple[bool, Optional[str]]:  # type: ignore[override]
+    def check_availability(  # type: ignore[override]  # Signature doesn't match base class
+        self,
+        stream: AbstractFileBasedStream,
+        logger: logging.Logger,
+        _: Optional[Source],
+    ) -> Tuple[bool, Optional[str]]:
         """
         Perform a connection check for the stream (verify that we can list files from the stream).
 
@@ -35,7 +49,10 @@ class DefaultFileBasedAvailabilityStrategy(AbstractFileBasedAvailabilityStrategy
         return True, None
 
     def check_availability_and_parsability(
-        self, stream: "AbstractFileBasedStream", logger: logging.Logger, _: Optional[Source]
+        self,
+        stream: AbstractFileBasedStream,
+        logger: logging.Logger,
+        _: Optional[Source],
     ) -> Tuple[bool, Optional[str]]:
         """
         Perform a connection check for the stream.
@@ -66,12 +83,14 @@ class DefaultFileBasedAvailabilityStrategy(AbstractFileBasedAvailabilityStrategy
                 # If the parser is set to not check parsability, we still want to check that we can open the file.
                 handle = stream.stream_reader.open_file(file, parser.file_read_mode, None, logger)
                 handle.close()
+        except AirbyteTracedException as ate:
+            raise ate
         except CheckAvailabilityError:
             return False, "".join(traceback.format_exc())
 
         return True, None
 
-    def _check_list_files(self, stream: "AbstractFileBasedStream") -> RemoteFile:
+    def _check_list_files(self, stream: AbstractFileBasedStream) -> RemoteFile:
         """
         Check that we can list files from the stream.
 
@@ -84,22 +103,39 @@ class DefaultFileBasedAvailabilityStrategy(AbstractFileBasedAvailabilityStrategy
         except CustomFileBasedException as exc:
             raise CheckAvailabilityError(str(exc), stream=stream.name) from exc
         except Exception as exc:
-            raise CheckAvailabilityError(FileBasedSourceError.ERROR_LISTING_FILES, stream=stream.name) from exc
+            raise CheckAvailabilityError(
+                FileBasedSourceError.ERROR_LISTING_FILES, stream=stream.name
+            ) from exc
 
         return file
 
-    def _check_parse_record(self, stream: "AbstractFileBasedStream", file: RemoteFile, logger: logging.Logger) -> None:
+    def _check_parse_record(
+        self,
+        stream: AbstractFileBasedStream,
+        file: RemoteFile,
+        logger: logging.Logger,
+    ) -> None:
         parser = stream.get_parser()
 
         try:
-            record = next(iter(parser.parse_records(stream.config, file, self.stream_reader, logger, discovered_schema=None)))
+            record = next(
+                iter(
+                    parser.parse_records(
+                        stream.config, file, self.stream_reader, logger, discovered_schema=None
+                    )
+                )
+            )
         except StopIteration:
             # The file is empty. We've verified that we can open it, so will
             # consider the connection check successful even though it means
             # we skip the schema validation check.
             return
+        except AirbyteTracedException as ate:
+            raise ate
         except Exception as exc:
-            raise CheckAvailabilityError(FileBasedSourceError.ERROR_READING_FILE, stream=stream.name, file=file.uri) from exc
+            raise CheckAvailabilityError(
+                FileBasedSourceError.ERROR_READING_FILE, stream=stream.name, file=file.uri
+            ) from exc
 
         schema = stream.catalog_schema or stream.config.input_schema
         if schema and stream.validation_policy.validate_schema_before_sync:

airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py

@@ -4,12 +4,43 @@
 
 import copy
 from abc import abstractmethod
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Literal, Optional, Union
 
-import dpath.util
+import dpath
+from pydantic.v1 import AnyUrl, BaseModel, Field
+
+from airbyte_cdk import OneOfOptionConfig
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.utils import schema_helpers
-from pydantic import AnyUrl, BaseModel, Field
+
+
+class DeliverRecords(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Replicate Records"
+        description = "Recommended - Extract and load structured records into your destination of choice. This is the classic method of moving data in Airbyte. It allows for blocking and hashing individual fields or files from a structured schema. Data can be flattened, typed and deduped depending on the destination."
+        discriminator = "delivery_type"
+
+    delivery_type: Literal["use_records_transfer"] = Field("use_records_transfer", const=True)
+
+
+class DeliverRawFiles(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Copy Raw Files"
+        description = "Copy raw files without parsing their contents. Bits are copied into the destination exactly as they appeared in the source. Recommended for use with unstructured text data, non-text and compressed files."
+        discriminator = "delivery_type"
+
+    delivery_type: Literal["use_file_transfer"] = Field("use_file_transfer", const=True)
+
+    preserve_directory_structure: bool = Field(
+        title="Preserve Sub-Directories in File Paths",
+        description=(
+            "If enabled, sends subdirectory folder structure "
+            "along with source file names to the destination. "
+            "Otherwise, files will be synced by their names only. "
+            "This option is ignored when file-based replication is not enabled."
+        ),
+        default=True,
+    )
 
 
 class AbstractFileBasedSpec(BaseModel):
@@ -34,6 +65,17 @@ class AbstractFileBasedSpec(BaseModel):
         order=10,
     )
 
+    delivery_method: Union[DeliverRecords, DeliverRawFiles] = Field(
+        title="Delivery Method",
+        discriminator="delivery_type",
+        type="object",
+        order=7,
+        display_type="radio",
+        group="advanced",
+        default="use_records_transfer",
+        airbyte_hidden=True,
+    )
+
     @classmethod
     @abstractmethod
     def documentation_url(cls) -> AnyUrl:
@@ -57,7 +99,7 @@ class AbstractFileBasedSpec(BaseModel):
     @staticmethod
     def remove_discriminator(schema: Dict[str, Any]) -> None:
         """pydantic adds "discriminator" to the schema for oneOfs, which is not treated right by the platform as we inline all references"""
-        dpath.util.delete(schema, "properties/**/discriminator")
+        dpath.delete(schema, "properties/**/discriminator")
 
     @staticmethod
     def replace_enum_allOf_and_anyOf(schema: Dict[str, Any]) -> Dict[str, Any]:
@@ -77,10 +119,16 @@ class AbstractFileBasedSpec(BaseModel):
 
         properties_to_change = ["validation_policy"]
         for property_to_change in properties_to_change:
-            property_object = schema["properties"]["streams"]["items"]["properties"][property_to_change]
+            property_object = schema["properties"]["streams"]["items"]["properties"][
+                property_to_change
+            ]
             if "anyOf" in property_object:
-                schema["properties"]["streams"]["items"]["properties"][property_to_change]["type"] = "object"
-                schema["properties"]["streams"]["items"]["properties"][property_to_change]["oneOf"] = property_object.pop("anyOf")
+                schema["properties"]["streams"]["items"]["properties"][property_to_change][
+                    "type"
+                ] = "object"
+                schema["properties"]["streams"]["items"]["properties"][property_to_change][
+                    "oneOf"
+                ] = property_object.pop("anyOf")
             AbstractFileBasedSpec.move_enum_to_root(property_object)
 
         csv_format_schemas = list(
@@ -91,9 +139,9 @@ class AbstractFileBasedSpec(BaseModel):
         )
         if len(csv_format_schemas) != 1:
             raise ValueError(f"Expecting only one CSV format but got {csv_format_schemas}")
-        csv_format_schemas[0]["properties"]["header_definition"]["oneOf"] = csv_format_schemas[0]["properties"]["header_definition"].pop(
-            "anyOf", []
-        )
+        csv_format_schemas[0]["properties"]["header_definition"]["oneOf"] = csv_format_schemas[0][
+            "properties"
+        ]["header_definition"].pop("anyOf", [])
         csv_format_schemas[0]["properties"]["header_definition"]["type"] = "object"
         return schema
 

airbyte_cdk/sources/file_based/config/avro_format.py

@@ -3,8 +3,9 @@
 #
 
 
+from pydantic.v1 import BaseModel, Field
+
 from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
-from pydantic import BaseModel, Field
 
 
 class AvroFormat(BaseModel):

airbyte_cdk/sources/file_based/config/csv_format.py

@@ -6,8 +6,10 @@ import codecs
 from enum import Enum
 from typing import Any, Dict, List, Optional, Set, Union
 
+from pydantic.v1 import BaseModel, Field, root_validator, validator
+from pydantic.v1.error_wrappers import ValidationError
+
 from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
-from pydantic import BaseModel, Field, ValidationError, root_validator, validator
 
 
 class InferenceType(Enum):
@@ -69,7 +71,9 @@ class CsvHeaderUserProvided(BaseModel):
     @validator("column_names")
     def validate_column_names(cls, v: List[str]) -> List[str]:
         if not v:
-            raise ValueError("At least one column name needs to be provided when using user provided headers")
+            raise ValueError(
+                "At least one column name needs to be provided when using user provided headers"
+            )
         return v
 
 
@@ -106,7 +110,9 @@ class CsvFormat(BaseModel):
         description='The character encoding of the CSV data. Leave blank to default to <strong>UTF8</strong>. See <a href="https://docs.python.org/3/library/codecs.html#standard-encodings" target="_blank">list of python encodings</a> for allowable options.',
     )
     double_quote: bool = Field(
-        title="Double Quote", default=True, description="Whether two quotes in a quoted CSV value denote a single quote in the data."
+        title="Double Quote",
+        default=True,
+        description="Whether two quotes in a quoted CSV value denote a single quote in the data.",
     )
     null_values: Set[str] = Field(
         title="Null Values",
@@ -124,12 +130,16 @@ class CsvFormat(BaseModel):
         description="The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.",
     )
     skip_rows_after_header: int = Field(
-        title="Skip Rows After Header", default=0, description="The number of rows to skip after the header row."
+        title="Skip Rows After Header",
+        default=0,
+        description="The number of rows to skip after the header row.",
     )
-    header_definition: Union[CsvHeaderFromCsv, CsvHeaderAutogenerated, CsvHeaderUserProvided] = Field(
-        title="CSV Header Definition",
-        default=CsvHeaderFromCsv(header_definition_type=CsvHeaderDefinitionType.FROM_CSV.value),
-        description="How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.",
+    header_definition: Union[CsvHeaderFromCsv, CsvHeaderAutogenerated, CsvHeaderUserProvided] = (
+        Field(
+            title="CSV Header Definition",
+            default=CsvHeaderFromCsv(header_definition_type=CsvHeaderDefinitionType.FROM_CSV.value),
+            description="How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.",
+        )
     )
     true_values: Set[str] = Field(
         title="True Values",
@@ -147,6 +157,11 @@ class CsvFormat(BaseModel):
         description="How to infer the types of the columns. If none, inference default to strings.",
         airbyte_hidden=True,
     )
+    ignore_errors_on_fields_mismatch: bool = Field(
+        title="Ignore errors on field mismatch",
+        default=False,
+        description="Whether to ignore errors that occur when the number of fields in the CSV does not match the number of columns in the schema.",
+    )
 
     @validator("delimiter")
     def validate_delimiter(cls, v: str) -> str:
@@ -183,9 +198,13 @@ class CsvFormat(BaseModel):
         definition_type = values.get("header_definition_type")
         column_names = values.get("user_provided_column_names")
         if definition_type == CsvHeaderDefinitionType.USER_PROVIDED and not column_names:
-            raise ValidationError("`user_provided_column_names` should be defined if the definition 'User Provided'.", model=CsvFormat)
+            raise ValidationError(
+                "`user_provided_column_names` should be defined if the definition 'User Provided'.",
+                model=CsvFormat,
+            )
         if definition_type != CsvHeaderDefinitionType.USER_PROVIDED and column_names:
             raise ValidationError(
-                "`user_provided_column_names` should not be defined if the definition is not 'User Provided'.", model=CsvFormat
+                "`user_provided_column_names` should not be defined if the definition is not 'User Provided'.",
+                model=CsvFormat,
             )
         return values

airbyte_cdk/sources/file_based/config/excel_format.py

@@ -0,0 +1,18 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from pydantic.v1 import BaseModel, Field
+
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
+
+
+class ExcelFormat(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Excel Format"
+        discriminator = "filetype"
+
+    filetype: str = Field(
+        "excel",
+        const=True,
+    )

airbyte_cdk/sources/file_based/config/file_based_stream_config.py

@@ -1,18 +1,20 @@
 #
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 #
 
 from enum import Enum
 from typing import Any, List, Mapping, Optional, Union
 
+from pydantic.v1 import BaseModel, Field, validator
+
 from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat
 from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat
+from airbyte_cdk.sources.file_based.config.excel_format import ExcelFormat
 from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat
 from airbyte_cdk.sources.file_based.config.parquet_format import ParquetFormat
 from airbyte_cdk.sources.file_based.config.unstructured_format import UnstructuredFormat
 from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError
 from airbyte_cdk.sources.file_based.schema_helpers import type_mapping_to_jsonschema
-from pydantic import BaseModel, Field, validator
 
 PrimaryKeyType = Optional[Union[str, List[str]]]
 
@@ -55,7 +57,9 @@ class FileBasedStreamConfig(BaseModel):
         description="When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.",
         default=3,
     )
-    format: Union[AvroFormat, CsvFormat, JsonlFormat, ParquetFormat, UnstructuredFormat] = Field(
+    format: Union[
+        AvroFormat, CsvFormat, JsonlFormat, ParquetFormat, UnstructuredFormat, ExcelFormat
+    ] = Field(
         title="Format",
         description="The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.",
     )
@@ -64,6 +68,12 @@ class FileBasedStreamConfig(BaseModel):
         description="When enabled, syncs will not validate or structure records against the stream's schema.",
         default=False,
     )
+    recent_n_files_to_read_for_schema_discovery: Optional[int] = Field(
+        title="Files To Read For Schema Discover",
+        description="The number of resent files which will be used to discover the schema for this stream.",
+        default=None,
+        gt=0,
+    )
 
     @validator("input_schema", pre=True)
     def validate_input_schema(cls, v: Optional[str]) -> Optional[str]:
@@ -82,6 +92,8 @@ class FileBasedStreamConfig(BaseModel):
         if self.input_schema:
             schema = type_mapping_to_jsonschema(self.input_schema)
             if not schema:
-                raise ValueError(f"Unable to create JSON schema from input schema {self.input_schema}")
+                raise ValueError(
+                    f"Unable to create JSON schema from input schema {self.input_schema}"
+                )
             return schema
         return None

airbyte_cdk/sources/file_based/config/jsonl_format.py

@@ -2,8 +2,9 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+from pydantic.v1 import BaseModel, Field
+
 from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
-from pydantic import BaseModel, Field
 
 
 class JsonlFormat(BaseModel):

airbyte_cdk/sources/file_based/config/parquet_format.py

@@ -3,8 +3,9 @@
 #
 
 
+from pydantic.v1 import BaseModel, Field
+
 from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
-from pydantic import BaseModel, Field
 
 
 class ParquetFormat(BaseModel):

airbyte_cdk/sources/file_based/config/unstructured_format.py

@@ -4,8 +4,9 @@
 
 from typing import List, Literal, Optional, Union
 
+from pydantic.v1 import BaseModel, Field
+
 from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
-from pydantic import BaseModel, Field
 
 
 class LocalProcessingConfigModel(BaseModel):
@@ -13,7 +14,9 @@ class LocalProcessingConfigModel(BaseModel):
 
     class Config(OneOfOptionConfig):
         title = "Local"
-        description = "Process files locally, supporting `fast` and `ocr` modes. This is the default option."
+        description = (
+            "Process files locally, supporting `fast` and `ocr` modes. This is the default option."
+        )
         discriminator = "mode"
 
 
@@ -23,7 +26,9 @@ class APIParameterConfigModel(BaseModel):
         description="The name of the unstructured API parameter to use",
         examples=["combine_under_n_chars", "languages"],
     )
-    value: str = Field(title="Value", description="The value of the parameter", examples=["true", "hi_res"])
+    value: str = Field(
+        title="Value", description="The value of the parameter", examples=["true", "hi_res"]
+    )
 
 
 class APIProcessingConfigModel(BaseModel):
@@ -60,7 +65,7 @@ class APIProcessingConfigModel(BaseModel):
 
 class UnstructuredFormat(BaseModel):
     class Config(OneOfOptionConfig):
-        title = "Document File Type Format (Experimental)"
+        title = "Unstructured Document Format"
         description = "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file."
         discriminator = "filetype"
 
@@ -85,7 +90,10 @@ class UnstructuredFormat(BaseModel):
         description="The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf",
     )
 
-    processing: Union[LocalProcessingConfigModel, APIProcessingConfigModel,] = Field(
+    processing: Union[
+        LocalProcessingConfigModel,
+        APIProcessingConfigModel,
+    ] = Field(
         default=LocalProcessingConfigModel(mode="local"),
         title="Processing",
         description="Processing configuration",

airbyte_cdk/sources/file_based/discovery_policy/__init__.py

@@ -1,4 +1,8 @@
-from airbyte_cdk.sources.file_based.discovery_policy.abstract_discovery_policy import AbstractDiscoveryPolicy
-from airbyte_cdk.sources.file_based.discovery_policy.default_discovery_policy import DefaultDiscoveryPolicy
+from airbyte_cdk.sources.file_based.discovery_policy.abstract_discovery_policy import (
+    AbstractDiscoveryPolicy,
+)
+from airbyte_cdk.sources.file_based.discovery_policy.default_discovery_policy import (
+    DefaultDiscoveryPolicy,
+)
 
 __all__ = ["AbstractDiscoveryPolicy", "DefaultDiscoveryPolicy"]

airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py

@@ -15,9 +15,7 @@ class AbstractDiscoveryPolicy(ABC):
 
     @property
     @abstractmethod
-    def n_concurrent_requests(self) -> int:
-        ...
+    def n_concurrent_requests(self) -> int: ...
 
     @abstractmethod
-    def get_max_n_files_for_schema_inference(self, parser: FileTypeParser) -> int:
-        ...
+    def get_max_n_files_for_schema_inference(self, parser: FileTypeParser) -> int: ...

airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py

@@ -2,7 +2,9 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.file_based.discovery_policy.abstract_discovery_policy import AbstractDiscoveryPolicy
+from airbyte_cdk.sources.file_based.discovery_policy.abstract_discovery_policy import (
+    AbstractDiscoveryPolicy,
+)
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 
 DEFAULT_N_CONCURRENT_REQUESTS = 10
@@ -23,6 +25,9 @@ class DefaultDiscoveryPolicy(AbstractDiscoveryPolicy):
         return min(
             filter(
                 None,
-                (DEFAULT_MAX_N_FILES_FOR_STREAM_SCHEMA_INFERENCE, parser.parser_max_n_files_for_schema_inference),
+                (
+                    DEFAULT_MAX_N_FILES_FOR_STREAM_SCHEMA_INFERENCE,
+                    parser.parser_max_n_files_for_schema_inference,
+                ),
             )
         )