airbyte-cdk 6.5.3rc2__py3-none-any.whl → 6.5.5__py3-none-any.whl

airbyte_cdk/sources/file_based/file_types/parquet_parser.py

@@ -10,9 +10,19 @@ from urllib.parse import unquote
 
 import pyarrow as pa
 import pyarrow.parquet as pq
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ParquetFormat
-from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError, RecordParseError
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import (
+    FileBasedStreamConfig,
+    ParquetFormat,
+)
+from airbyte_cdk.sources.file_based.exceptions import (
+    ConfigValidationError,
+    FileBasedSourceError,
+    RecordParseError,
+)
+from airbyte_cdk.sources.file_based.file_based_stream_reader import (
+    AbstractFileBasedStreamReader,
+    FileReadMode,
+)
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.schema_helpers import SchemaType
@@ -20,7 +30,6 @@ from pyarrow import DictionaryArray, Scalar
 
 
 class ParquetParser(FileTypeParser):
-
     ENCODING = None
 
     def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[str]]:
@@ -45,9 +54,15 @@ class ParquetParser(FileTypeParser):
             parquet_schema = parquet_file.schema_arrow
 
         # Inferred non-partition schema
-        schema = {field.name: ParquetParser.parquet_type_to_schema_type(field.type, parquet_format) for field in parquet_schema}
+        schema = {
+            field.name: ParquetParser.parquet_type_to_schema_type(field.type, parquet_format)
+            for field in parquet_schema
+        }
         # Inferred partition schema
-        partition_columns = {partition.split("=")[0]: {"type": "string"} for partition in self._extract_partitions(file.uri)}
+        partition_columns = {
+            partition.split("=")[0]: {"type": "string"}
+            for partition in self._extract_partitions(file.uri)
+        }
 
         schema.update(partition_columns)
         return schema
@@ -69,21 +84,27 @@ class ParquetParser(FileTypeParser):
         try:
             with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
                 reader = pq.ParquetFile(fp)
-                partition_columns = {x.split("=")[0]: x.split("=")[1] for x in self._extract_partitions(file.uri)}
+                partition_columns = {
+                    x.split("=")[0]: x.split("=")[1] for x in self._extract_partitions(file.uri)
+                }
                 for row_group in range(reader.num_row_groups):
                     batch = reader.read_row_group(row_group)
                     for row in range(batch.num_rows):
                         line_no += 1
                         yield {
                             **{
-                                column: ParquetParser._to_output_value(batch.column(column)[row], parquet_format)
+                                column: ParquetParser._to_output_value(
+                                    batch.column(column)[row], parquet_format
+                                )
                                 for column in batch.column_names
                             },
                             **partition_columns,
                         }
         except Exception as exc:
             raise RecordParseError(
-                FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=f"{row_group=}, {line_no=}"
+                FileBasedSourceError.ERROR_PARSING_RECORD,
+                filename=file.uri,
+                lineno=f"{row_group=}, {line_no=}",
             ) from exc
 
     @staticmethod
@@ -95,7 +116,9 @@ class ParquetParser(FileTypeParser):
         return FileReadMode.READ_BINARY
 
     @staticmethod
-    def _to_output_value(parquet_value: Union[Scalar, DictionaryArray], parquet_format: ParquetFormat) -> Any:
+    def _to_output_value(
+        parquet_value: Union[Scalar, DictionaryArray], parquet_format: ParquetFormat
+    ) -> Any:
         """
         Convert an entry in a pyarrow table to a value that can be output by the source.
         """
@@ -113,7 +136,11 @@ class ParquetParser(FileTypeParser):
             return None
 
         # Convert date and datetime objects to isoformat strings
-        if pa.types.is_time(parquet_value.type) or pa.types.is_timestamp(parquet_value.type) or pa.types.is_date(parquet_value.type):
+        if (
+            pa.types.is_time(parquet_value.type)
+            or pa.types.is_timestamp(parquet_value.type)
+            or pa.types.is_date(parquet_value.type)
+        ):
             return parquet_value.as_py().isoformat()
 
         # Convert month_day_nano_interval to array
@@ -168,7 +195,9 @@ class ParquetParser(FileTypeParser):
         }
 
     @staticmethod
-    def parquet_type_to_schema_type(parquet_type: pa.DataType, parquet_format: ParquetFormat) -> Mapping[str, str]:
+    def parquet_type_to_schema_type(
+        parquet_type: pa.DataType, parquet_format: ParquetFormat
+    ) -> Mapping[str, str]:
         """
         Convert a pyarrow data type to an Airbyte schema type.
         Parquet data types are defined at https://arrow.apache.org/docs/python/api/datatypes.html
@@ -198,7 +227,9 @@ class ParquetParser(FileTypeParser):
     @staticmethod
     def _is_binary(parquet_type: pa.DataType) -> bool:
         return bool(
-            pa.types.is_binary(parquet_type) or pa.types.is_large_binary(parquet_type) or pa.types.is_fixed_size_binary(parquet_type)
+            pa.types.is_binary(parquet_type)
+            or pa.types.is_large_binary(parquet_type)
+            or pa.types.is_fixed_size_binary(parquet_type)
         )
 
     @staticmethod
@@ -221,13 +252,23 @@ class ParquetParser(FileTypeParser):
                 pa.types.is_time(parquet_type)
                 or pa.types.is_string(parquet_type)
                 or pa.types.is_large_string(parquet_type)
-                or ParquetParser._is_binary(parquet_type)  # Best we can do is return as a string since we do not support binary
+                or ParquetParser._is_binary(
+                    parquet_type
+                )  # Best we can do is return as a string since we do not support binary
             )
 
     @staticmethod
     def _is_object(parquet_type: pa.DataType) -> bool:
-        return bool(pa.types.is_dictionary(parquet_type) or pa.types.is_struct(parquet_type) or pa.types.is_map(parquet_type))
+        return bool(
+            pa.types.is_dictionary(parquet_type)
+            or pa.types.is_struct(parquet_type)
+            or pa.types.is_map(parquet_type)
+        )
 
     @staticmethod
     def _is_list(parquet_type: pa.DataType) -> bool:
-        return bool(pa.types.is_list(parquet_type) or pa.types.is_large_list(parquet_type) or parquet_type == pa.month_day_nano_interval())
+        return bool(
+            pa.types.is_list(parquet_type)
+            or pa.types.is_large_list(parquet_type)
+            or parquet_type == pa.month_day_nano_interval()
+        )

airbyte_cdk/sources/file_based/file_types/unstructured_parser.py

@@ -19,13 +19,21 @@ from airbyte_cdk.sources.file_based.config.unstructured_format import (
     UnstructuredFormat,
 )
 from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
+from airbyte_cdk.sources.file_based.file_based_stream_reader import (
+    AbstractFileBasedStreamReader,
+    FileReadMode,
+)
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.schema_helpers import SchemaType
 from airbyte_cdk.utils import is_cloud_environment
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from unstructured.file_utils.filetype import FILETYPE_TO_MIMETYPE, STR_TO_FILETYPE, FileType, detect_filetype
+from unstructured.file_utils.filetype import (
+    FILETYPE_TO_MIMETYPE,
+    STR_TO_FILETYPE,
+    FileType,
+    detect_filetype,
+)
 
 unstructured_partition_pdf = None
 unstructured_partition_docx = None
@@ -109,7 +117,10 @@ class UnstructuredParser(FileTypeParser):
                     "type": "string",
                     "description": "Content of the file as markdown. Might be null if the file could not be parsed",
                 },
-                "document_key": {"type": "string", "description": "Unique identifier of the document, e.g. the file path"},
+                "document_key": {
+                    "type": "string",
+                    "description": "Unique identifier of the document, e.g. the file path",
+                },
                 "_ab_source_file_parse_error": {
                     "type": "string",
                     "description": "Error message if the file could not be parsed even though the file is supported",
@@ -149,9 +160,19 @@ class UnstructuredParser(FileTypeParser):
                 else:
                     raise e
 
-    def _read_file(self, file_handle: IOBase, remote_file: RemoteFile, format: UnstructuredFormat, logger: logging.Logger) -> str:
+    def _read_file(
+        self,
+        file_handle: IOBase,
+        remote_file: RemoteFile,
+        format: UnstructuredFormat,
+        logger: logging.Logger,
+    ) -> str:
         _import_unstructured()
-        if (not unstructured_partition_pdf) or (not unstructured_partition_docx) or (not unstructured_partition_pptx):
+        if (
+            (not unstructured_partition_pdf)
+            or (not unstructured_partition_docx)
+            or (not unstructured_partition_pptx)
+        ):
             # check whether unstructured library is actually available for better error message and to ensure proper typing (can't be None after this point)
             raise Exception("unstructured library is not available")
 
@@ -167,7 +188,9 @@ class UnstructuredParser(FileTypeParser):
             return self._read_file_locally(file_handle, filetype, format.strategy, remote_file)
         elif format.processing.mode == "api":
             try:
-                result: str = self._read_file_remotely_with_retries(file_handle, format.processing, filetype, format.strategy, remote_file)
+                result: str = self._read_file_remotely_with_retries(
+                    file_handle, format.processing, filetype, format.strategy, remote_file
+                )
             except Exception as e:
                 # If a parser error happens during remotely processing the file, this means the file is corrupted. This case is handled by the parse_records method, so just rethrow.
                 #
@@ -175,11 +198,15 @@ class UnstructuredParser(FileTypeParser):
                 # Once this parser leaves experimental stage, we should consider making this a system error instead for issues that might be transient.
                 if isinstance(e, RecordParseError):
                     raise e
-                raise AirbyteTracedException.from_exception(e, failure_type=FailureType.config_error)
+                raise AirbyteTracedException.from_exception(
+                    e, failure_type=FailureType.config_error
+                )
 
             return result
 
-    def _params_to_dict(self, params: Optional[List[APIParameterConfigModel]], strategy: str) -> Dict[str, Union[str, List[str]]]:
+    def _params_to_dict(
+        self, params: Optional[List[APIParameterConfigModel]], strategy: str
+    ) -> Dict[str, Union[str, List[str]]]:
         result_dict: Dict[str, Union[str, List[str]]] = {"strategy": strategy}
         if params is None:
             return result_dict
@@ -229,9 +256,16 @@ class UnstructuredParser(FileTypeParser):
 
         return True, None
 
-    @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=5, giveup=user_error)
+    @backoff.on_exception(
+        backoff.expo, requests.exceptions.RequestException, max_tries=5, giveup=user_error
+    )
     def _read_file_remotely_with_retries(
-        self, file_handle: IOBase, format: APIProcessingConfigModel, filetype: FileType, strategy: str, remote_file: RemoteFile
+        self,
+        file_handle: IOBase,
+        format: APIProcessingConfigModel,
+        filetype: FileType,
+        strategy: str,
+        remote_file: RemoteFile,
     ) -> str:
         """
         Read a file remotely, retrying up to 5 times if the error is not caused by user error. This is useful for transient network errors or the API server being overloaded temporarily.
@@ -239,7 +273,12 @@ class UnstructuredParser(FileTypeParser):
         return self._read_file_remotely(file_handle, format, filetype, strategy, remote_file)
 
     def _read_file_remotely(
-        self, file_handle: IOBase, format: APIProcessingConfigModel, filetype: FileType, strategy: str, remote_file: RemoteFile
+        self,
+        file_handle: IOBase,
+        format: APIProcessingConfigModel,
+        filetype: FileType,
+        strategy: str,
+        remote_file: RemoteFile,
     ) -> str:
         headers = {"accept": "application/json", "unstructured-api-key": format.api_key}
 
@@ -247,7 +286,9 @@ class UnstructuredParser(FileTypeParser):
 
         file_data = {"files": ("filename", file_handle, FILETYPE_TO_MIMETYPE[filetype])}
 
-        response = requests.post(f"{format.api_url}/general/v0/general", headers=headers, data=data, files=file_data)
+        response = requests.post(
+            f"{format.api_url}/general/v0/general", headers=headers, data=data, files=file_data
+        )
 
         if response.status_code == 422:
             # 422 means the file couldn't be processed, but the API is working. Treat this as a parsing error (passing an error record to the destination).
@@ -260,9 +301,15 @@ class UnstructuredParser(FileTypeParser):
 
         return self._render_markdown(json_response)
 
-    def _read_file_locally(self, file_handle: IOBase, filetype: FileType, strategy: str, remote_file: RemoteFile) -> str:
+    def _read_file_locally(
+        self, file_handle: IOBase, filetype: FileType, strategy: str, remote_file: RemoteFile
+    ) -> str:
         _import_unstructured()
-        if (not unstructured_partition_pdf) or (not unstructured_partition_docx) or (not unstructured_partition_pptx):
+        if (
+            (not unstructured_partition_pdf)
+            or (not unstructured_partition_docx)
+            or (not unstructured_partition_pptx)
+        ):
             # check whether unstructured library is actually available for better error message and to ensure proper typing (can't be None after this point)
             raise Exception("unstructured library is not available")
 
@@ -290,7 +337,9 @@ class UnstructuredParser(FileTypeParser):
         return self._render_markdown([element.to_dict() for element in elements])
 
     def _create_parse_error(self, remote_file: RemoteFile, message: str) -> RecordParseError:
-        return RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=remote_file.uri, message=message)
+        return RecordParseError(
+            FileBasedSourceError.ERROR_PARSING_RECORD, filename=remote_file.uri, message=message
+        )
 
     def _get_filetype(self, file: IOBase, remote_file: RemoteFile) -> Optional[FileType]:
         """

airbyte_cdk/sources/file_based/schema_helpers.py

@@ -8,13 +8,20 @@ from enum import Enum
 from functools import total_ordering
 from typing import Any, Dict, List, Literal, Mapping, Optional, Tuple, Type, Union
 
-from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError, SchemaInferenceError
+from airbyte_cdk.sources.file_based.exceptions import (
+    ConfigValidationError,
+    FileBasedSourceError,
+    SchemaInferenceError,
+)
 
 JsonSchemaSupportedType = Union[List[str], Literal["string"], str]
 SchemaType = Mapping[str, Mapping[str, JsonSchemaSupportedType]]
 
 schemaless_schema = {"type": "object", "properties": {"data": {"type": "object"}}}
-file_transfer_schema = {"type": "object", "properties": {"data": {"type": "object"}, "file": {"type": "object"}}}
+file_transfer_schema = {
+    "type": "object",
+    "properties": {"data": {"type": "object"}, "file": {"type": "object"}},
+}
 
 
 @total_ordering
@@ -129,7 +136,12 @@ def _choose_wider_type(key: str, t1: Mapping[str, Any], t2: Mapping[str, Any]) -
             detected_types=f"{t1},{t2}",
         )
     # Schemas can still be merged if a key contains a null value in either t1 or t2, but it is still an object
-    elif (t1_type == "object" or t2_type == "object") and t1_type != "null" and t2_type != "null" and t1 != t2:
+    elif (
+        (t1_type == "object" or t2_type == "object")
+        and t1_type != "null"
+        and t2_type != "null"
+        and t1 != t2
+    ):
         raise SchemaInferenceError(
             FileBasedSourceError.SCHEMA_INFERENCE_ERROR,
             details="Cannot merge schema for unequal object types.",
@@ -137,12 +149,19 @@ def _choose_wider_type(key: str, t1: Mapping[str, Any], t2: Mapping[str, Any]) -
             detected_types=f"{t1},{t2}",
         )
     else:
-        comparable_t1 = get_comparable_type(TYPE_PYTHON_MAPPING[t1_type][0])  # accessing the type_mapping value
-        comparable_t2 = get_comparable_type(TYPE_PYTHON_MAPPING[t2_type][0])  # accessing the type_mapping value
+        comparable_t1 = get_comparable_type(
+            TYPE_PYTHON_MAPPING[t1_type][0]
+        )  # accessing the type_mapping value
+        comparable_t2 = get_comparable_type(
+            TYPE_PYTHON_MAPPING[t2_type][0]
+        )  # accessing the type_mapping value
         if not comparable_t1 and comparable_t2:
-            raise SchemaInferenceError(FileBasedSourceError.UNRECOGNIZED_TYPE, key=key, detected_types=f"{t1},{t2}")
+            raise SchemaInferenceError(
+                FileBasedSourceError.UNRECOGNIZED_TYPE, key=key, detected_types=f"{t1},{t2}"
+            )
         return max(
-            [t1, t2], key=lambda x: ComparableType(get_comparable_type(TYPE_PYTHON_MAPPING[x["type"]][0]))
+            [t1, t2],
+            key=lambda x: ComparableType(get_comparable_type(TYPE_PYTHON_MAPPING[x["type"]][0])),
         )  # accessing the type_mapping value
 
 
@@ -205,7 +224,8 @@ def _parse_json_input(input_schema: Union[str, Mapping[str, str]]) -> Optional[M
             schema = input_schema
         if not all(isinstance(s, str) for s in schema.values()):
             raise ConfigValidationError(
-                FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA, details="Invalid input schema; nested schemas are not supported."
+                FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA,
+                details="Invalid input schema; nested schemas are not supported.",
             )
 
     except json.decoder.JSONDecodeError:
@@ -214,7 +234,9 @@ def _parse_json_input(input_schema: Union[str, Mapping[str, str]]) -> Optional[M
     return schema
 
 
-def type_mapping_to_jsonschema(input_schema: Optional[Union[str, Mapping[str, str]]]) -> Optional[Mapping[str, Any]]:
+def type_mapping_to_jsonschema(
+    input_schema: Optional[Union[str, Mapping[str, str]]],
+) -> Optional[Mapping[str, Any]]:
     """
     Return the user input schema (type mapping), transformed to JSON Schema format.
 
@@ -241,7 +263,8 @@ def type_mapping_to_jsonschema(input_schema: Optional[Union[str, Mapping[str, st
 
         if not _json_schema_type:
             raise ConfigValidationError(
-                FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA, details=f"Invalid type '{type_name}' for property '{col_name}'."
+                FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA,
+                details=f"Invalid type '{type_name}' for property '{col_name}'.",
             )
 
         json_schema_type = _json_schema_type[0]

airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py

@@ -11,7 +11,9 @@ class AbstractSchemaValidationPolicy(ABC):
     validate_schema_before_sync = False  # Whether to verify that records conform to the schema during the stream's availabilty check
 
     @abstractmethod
-    def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool:
+    def record_passes_validation_policy(
+        self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]
+    ) -> bool:
         """
         Return True if the record passes the user's validation policy.
         """

airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py

@@ -5,7 +5,10 @@
 from typing import Any, Mapping, Optional
 
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import ValidationPolicy
-from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, StopSyncPerValidationPolicy
+from airbyte_cdk.sources.file_based.exceptions import (
+    FileBasedSourceError,
+    StopSyncPerValidationPolicy,
+)
 from airbyte_cdk.sources.file_based.schema_helpers import conforms_to_schema
 from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy
 
@@ -13,14 +16,18 @@ from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSc
 class EmitRecordPolicy(AbstractSchemaValidationPolicy):
     name = "emit_record"
 
-    def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool:
+    def record_passes_validation_policy(
+        self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]
+    ) -> bool:
         return True
 
 
 class SkipRecordPolicy(AbstractSchemaValidationPolicy):
     name = "skip_record"
 
-    def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool:
+    def record_passes_validation_policy(
+        self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]
+    ) -> bool:
         return schema is not None and conforms_to_schema(record, schema)
 
 
@@ -28,9 +35,13 @@ class WaitForDiscoverPolicy(AbstractSchemaValidationPolicy):
     name = "wait_for_discover"
     validate_schema_before_sync = True
 
-    def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool:
+    def record_passes_validation_policy(
+        self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]
+    ) -> bool:
         if schema is None or not conforms_to_schema(record, schema):
-            raise StopSyncPerValidationPolicy(FileBasedSourceError.STOP_SYNC_PER_SCHEMA_VALIDATION_POLICY)
+            raise StopSyncPerValidationPolicy(
+                FileBasedSourceError.STOP_SYNC_PER_SCHEMA_VALIDATION_POLICY
+            )
         return True
 
 

airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py

@@ -8,10 +8,20 @@ from typing import Any, Dict, Iterable, List, Mapping, Optional, Type
 
 from airbyte_cdk import AirbyteMessage
 from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, PrimaryKeyType
+from airbyte_cdk.sources.file_based.availability_strategy import (
+    AbstractFileBasedAvailabilityStrategy,
+)
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import (
+    FileBasedStreamConfig,
+    PrimaryKeyType,
+)
 from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy
-from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector, FileBasedSourceError, RecordParseError, UndefinedParserError
+from airbyte_cdk.sources.file_based.exceptions import (
+    FileBasedErrorsCollector,
+    FileBasedSourceError,
+    RecordParseError,
+    UndefinedParserError,
+)
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
@@ -64,8 +74,7 @@ class AbstractFileBasedStream(Stream):
 
     @property
     @abstractmethod
-    def primary_key(self) -> PrimaryKeyType:
-        ...
+    def primary_key(self) -> PrimaryKeyType: ...
 
     @cache
     def list_files(self) -> List[RemoteFile]:
@@ -102,14 +111,20 @@ class AbstractFileBasedStream(Stream):
         return self.read_records_from_slice(stream_slice)
 
     @abstractmethod
-    def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
+    def read_records_from_slice(
+        self, stream_slice: StreamSlice
+    ) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
         """
         Yield all records from all remote files in `list_files_for_this_sync`.
         """
         ...
 
     def stream_slices(
-        self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None
+        self,
+        *,
+        sync_mode: SyncMode,
+        cursor_field: Optional[List[str]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[Optional[Mapping[str, Any]]]:
         """
         This method acts as an adapter between the generic Stream interface and the file-based's
@@ -144,14 +159,22 @@ class AbstractFileBasedStream(Stream):
         try:
             return self._parsers[type(self.config.format)]
         except KeyError:
-            raise UndefinedParserError(FileBasedSourceError.UNDEFINED_PARSER, stream=self.name, format=type(self.config.format))
+            raise UndefinedParserError(
+                FileBasedSourceError.UNDEFINED_PARSER,
+                stream=self.name,
+                format=type(self.config.format),
+            )
 
     def record_passes_validation_policy(self, record: Mapping[str, Any]) -> bool:
         if self.validation_policy:
-            return self.validation_policy.record_passes_validation_policy(record=record, schema=self.catalog_schema)
+            return self.validation_policy.record_passes_validation_policy(
+                record=record, schema=self.catalog_schema
+            )
         else:
             raise RecordParseError(
-                FileBasedSourceError.UNDEFINED_VALIDATION_POLICY, stream=self.name, validation_policy=self.config.validation_policy
+                FileBasedSourceError.UNDEFINED_VALIDATION_POLICY,
+                stream=self.name,
+                validation_policy=self.config.validation_policy,
             )
 
     @cached_property

airbyte_cdk/sources/file_based/stream/concurrent/adapters.py

@@ -7,7 +7,14 @@ import logging
 from functools import cache, lru_cache
 from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteStream, Level, SyncMode, Type
+from airbyte_cdk.models import (
+    AirbyteLogMessage,
+    AirbyteMessage,
+    ConfiguredAirbyteStream,
+    Level,
+    SyncMode,
+    Type,
+)
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.file_based.availability_strategy import (
@@ -26,7 +33,10 @@ from airbyte_cdk.sources.source import ExperimentalClassWarning
 from airbyte_cdk.sources.streams.concurrent.abstract_stream_facade import AbstractStreamFacade
 from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage
-from airbyte_cdk.sources.streams.concurrent.helpers import get_cursor_field_from_stream, get_primary_key_from_stream
+from airbyte_cdk.sources.streams.concurrent.helpers import (
+    get_cursor_field_from_stream,
+    get_primary_key_from_stream,
+)
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
 from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
 from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
@@ -36,7 +46,9 @@ from airbyte_cdk.sources.utils.slice_logger import SliceLogger
 from deprecated.classic import deprecated
 
 if TYPE_CHECKING:
-    from airbyte_cdk.sources.file_based.stream.concurrent.cursor import AbstractConcurrentFileBasedCursor
+    from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
+        AbstractConcurrentFileBasedCursor,
+    )
 
 """
 This module contains adapters to help enabling concurrency on File-based Stream objects without needing to migrate to AbstractStream
@@ -72,7 +84,9 @@ class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBas
                 partition_generator=FileBasedStreamPartitionGenerator(
                     stream,
                     message_repository,
-                    SyncMode.full_refresh if isinstance(cursor, FileBasedFinalStateCursor) else SyncMode.incremental,
+                    SyncMode.full_refresh
+                    if isinstance(cursor, FileBasedFinalStateCursor)
+                    else SyncMode.incremental,
                     [cursor_field] if cursor_field is not None else None,
                     state,
                     cursor,
@@ -138,7 +152,10 @@ class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBas
 
     @property
     def primary_key(self) -> PrimaryKeyType:
-        return self._legacy_stream.config.primary_key or self.get_parser().get_parser_defined_primary_key(self._legacy_stream.config)
+        return (
+            self._legacy_stream.config.primary_key
+            or self.get_parser().get_parser_defined_primary_key(self._legacy_stream.config)
+        )
 
     def get_parser(self) -> FileTypeParser:
         return self._legacy_stream.get_parser()
@@ -185,7 +202,10 @@ class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBas
                 # This shouldn't happen if the ConcurrentCursor was used
                 state = "unknown; no state attribute was available on the cursor"
             yield AirbyteMessage(
-                type=Type.LOG, log=AirbyteLogMessage(level=Level.ERROR, message=f"Cursor State at time of exception: {state}")
+                type=Type.LOG,
+                log=AirbyteLogMessage(
+                    level=Level.ERROR, message=f"Cursor State at time of exception: {state}"
+                ),
             )
             raise exc
 
@@ -227,16 +247,30 @@ class FileBasedStreamPartition(Partition):
             ):
                 if isinstance(record_data, Mapping):
                     data_to_return = dict(record_data)
-                    self._stream.transformer.transform(data_to_return, self._stream.get_json_schema())
+                    self._stream.transformer.transform(
+                        data_to_return, self._stream.get_json_schema()
+                    )
                     yield Record(data_to_return, self)
-                elif isinstance(record_data, AirbyteMessage) and record_data.type == Type.RECORD and record_data.record is not None:
+                elif (
+                    isinstance(record_data, AirbyteMessage)
+                    and record_data.type == Type.RECORD
+                    and record_data.record is not None
+                ):
                     # `AirbyteMessage`s of type `Record` should also be yielded so they are enqueued
                     # If stream is flagged for file_transfer the record should data in file key
-                    record_message_data = record_data.record.file if self._use_file_transfer() else record_data.record.data
+                    record_message_data = (
+                        record_data.record.file
+                        if self._use_file_transfer()
+                        else record_data.record.data
+                    )
                     if not record_message_data:
                         raise ExceptionWithDisplayMessage("A record without data was found")
                     else:
-                        yield Record(data=record_message_data, partition=self, is_file_transfer_message=self._use_file_transfer())
+                        yield Record(
+                            data=record_message_data,
+                            partition=self,
+                            is_file_transfer_message=self._use_file_transfer(),
+                        )
                 else:
                     self._message_repository.emit_message(record_data)
         except Exception as e:
@@ -305,7 +339,9 @@ class FileBasedStreamPartitionGenerator(PartitionGenerator):
 
     def generate(self) -> Iterable[FileBasedStreamPartition]:
         pending_partitions = []
-        for _slice in self._stream.stream_slices(sync_mode=self._sync_mode, cursor_field=self._cursor_field, stream_state=self._state):
+        for _slice in self._stream.stream_slices(
+            sync_mode=self._sync_mode, cursor_field=self._cursor_field, stream_state=self._state
+        ):
             if _slice is not None:
                 for file in _slice.get("files", []):
                     pending_partitions.append(