airbyte-cdk 0.72.1__py3-none-any.whl → 6.13.1.dev4107__py3-none-any.whl

airbyte_cdk/sources/file_based/file_types/file_transfer.py

@@ -0,0 +1,37 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+import logging
+import os
+from typing import Any, Dict, Iterable
+
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
+from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+
+AIRBYTE_STAGING_DIRECTORY = os.getenv("AIRBYTE_STAGING_DIRECTORY", "/staging/files")
+DEFAULT_LOCAL_DIRECTORY = "/tmp/airbyte-file-transfer"
+
+
+class FileTransfer:
+    def __init__(self) -> None:
+        self._local_directory = (
+            AIRBYTE_STAGING_DIRECTORY
+            if os.path.exists(AIRBYTE_STAGING_DIRECTORY)
+            else DEFAULT_LOCAL_DIRECTORY
+        )
+
+    def get_file(
+        self,
+        config: FileBasedStreamConfig,
+        file: RemoteFile,
+        stream_reader: AbstractFileBasedStreamReader,
+        logger: logging.Logger,
+    ) -> Iterable[Dict[str, Any]]:
+        try:
+            yield stream_reader.get_file(
+                file=file, local_directory=self._local_directory, logger=logger
+            )
+        except Exception as ex:
+            logger.error("An error has occurred while getting file: %s", str(ex))
+            raise ex

airbyte_cdk/sources/file_based/file_types/file_type_parser.py

@@ -7,7 +7,10 @@ from abc import ABC, abstractmethod
 from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
 
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
+from airbyte_cdk.sources.file_based.file_based_stream_reader import (
+    AbstractFileBasedStreamReader,
+    FileReadMode,
+)
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.schema_helpers import SchemaType
 

airbyte_cdk/sources/file_based/file_types/jsonl_parser.py

@@ -6,16 +6,24 @@ import json
 import logging
 from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Union
 
+import orjson
+
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
+from airbyte_cdk.sources.file_based.file_based_stream_reader import (
+    AbstractFileBasedStreamReader,
+    FileReadMode,
+)
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.file_based.schema_helpers import PYTHON_TYPE_MAPPING, SchemaType, merge_schemas
+from airbyte_cdk.sources.file_based.schema_helpers import (
+    PYTHON_TYPE_MAPPING,
+    SchemaType,
+    merge_schemas,
+)
 
 
 class JsonlParser(FileTypeParser):
-
     MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE = 1_000_000
     ENCODING = "utf8"
 
@@ -100,18 +108,24 @@ class JsonlParser(FileTypeParser):
                 read_bytes += len(line)
                 accumulator += line  # type: ignore [operator]  # In reality, it's either bytes or string and we add the same type
                 try:
-                    record = json.loads(accumulator)
+                    record = orjson.loads(accumulator)
                     if had_json_parsing_error and not has_warned_for_multiline_json_object:
-                        logger.warning(f"File at {file.uri} is using multiline JSON. Performance could be greatly reduced")
+                        logger.warning(
+                            f"File at {file.uri} is using multiline JSON. Performance could be greatly reduced"
+                        )
                         has_warned_for_multiline_json_object = True
 
                     yield record
                     yielded_at_least_once = True
                     accumulator = self._instantiate_accumulator(line)
-                except json.JSONDecodeError:
+                except orjson.JSONDecodeError:
                     had_json_parsing_error = True
 
-                if read_limit and yielded_at_least_once and read_bytes >= self.MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE:
+                if (
+                    read_limit
+                    and yielded_at_least_once
+                    and read_bytes >= self.MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE
+                ):
                     logger.warning(
                         f"Exceeded the maximum number of bytes per file for schema inference ({self.MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE}). "
                         f"Inferring schema from an incomplete set of records."
@@ -119,7 +133,9 @@ class JsonlParser(FileTypeParser):
                     break
 
             if had_json_parsing_error and not yielded_at_least_once:
-                raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line)
+                raise RecordParseError(
+                    FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line
+                )
 
     @staticmethod
     def _instantiate_accumulator(line: Union[bytes, str]) -> Union[bytes, str]:

airbyte_cdk/sources/file_based/file_types/parquet_parser.py

@@ -10,17 +10,27 @@ from urllib.parse import unquote
 
 import pyarrow as pa
 import pyarrow.parquet as pq
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ParquetFormat
-from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError, RecordParseError
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
+from pyarrow import DictionaryArray, Scalar
+
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import (
+    FileBasedStreamConfig,
+    ParquetFormat,
+)
+from airbyte_cdk.sources.file_based.exceptions import (
+    ConfigValidationError,
+    FileBasedSourceError,
+    RecordParseError,
+)
+from airbyte_cdk.sources.file_based.file_based_stream_reader import (
+    AbstractFileBasedStreamReader,
+    FileReadMode,
+)
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.schema_helpers import SchemaType
-from pyarrow import DictionaryArray, Scalar
 
 
 class ParquetParser(FileTypeParser):
-
     ENCODING = None
 
     def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[str]]:
@@ -45,9 +55,15 @@ class ParquetParser(FileTypeParser):
             parquet_schema = parquet_file.schema_arrow
 
         # Inferred non-partition schema
-        schema = {field.name: ParquetParser.parquet_type_to_schema_type(field.type, parquet_format) for field in parquet_schema}
+        schema = {
+            field.name: ParquetParser.parquet_type_to_schema_type(field.type, parquet_format)
+            for field in parquet_schema
+        }
         # Inferred partition schema
-        partition_columns = {partition.split("=")[0]: {"type": "string"} for partition in self._extract_partitions(file.uri)}
+        partition_columns = {
+            partition.split("=")[0]: {"type": "string"}
+            for partition in self._extract_partitions(file.uri)
+        }
 
         schema.update(partition_columns)
         return schema
@@ -69,21 +85,27 @@ class ParquetParser(FileTypeParser):
         try:
             with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
                 reader = pq.ParquetFile(fp)
-                partition_columns = {x.split("=")[0]: x.split("=")[1] for x in self._extract_partitions(file.uri)}
+                partition_columns = {
+                    x.split("=")[0]: x.split("=")[1] for x in self._extract_partitions(file.uri)
+                }
                 for row_group in range(reader.num_row_groups):
                     batch = reader.read_row_group(row_group)
                     for row in range(batch.num_rows):
                         line_no += 1
                         yield {
                             **{
-                                column: ParquetParser._to_output_value(batch.column(column)[row], parquet_format)
+                                column: ParquetParser._to_output_value(
+                                    batch.column(column)[row], parquet_format
+                                )
                                 for column in batch.column_names
                             },
                             **partition_columns,
                         }
         except Exception as exc:
             raise RecordParseError(
-                FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=f"{row_group=}, {line_no=}"
+                FileBasedSourceError.ERROR_PARSING_RECORD,
+                filename=file.uri,
+                lineno=f"{row_group=}, {line_no=}",
             ) from exc
 
     @staticmethod
@@ -95,7 +117,9 @@ class ParquetParser(FileTypeParser):
         return FileReadMode.READ_BINARY
 
     @staticmethod
-    def _to_output_value(parquet_value: Union[Scalar, DictionaryArray], parquet_format: ParquetFormat) -> Any:
+    def _to_output_value(
+        parquet_value: Union[Scalar, DictionaryArray], parquet_format: ParquetFormat
+    ) -> Any:
         """
         Convert an entry in a pyarrow table to a value that can be output by the source.
         """
@@ -113,7 +137,11 @@ class ParquetParser(FileTypeParser):
             return None
 
         # Convert date and datetime objects to isoformat strings
-        if pa.types.is_time(parquet_value.type) or pa.types.is_timestamp(parquet_value.type) or pa.types.is_date(parquet_value.type):
+        if (
+            pa.types.is_time(parquet_value.type)
+            or pa.types.is_timestamp(parquet_value.type)
+            or pa.types.is_date(parquet_value.type)
+        ):
             return parquet_value.as_py().isoformat()
 
         # Convert month_day_nano_interval to array
@@ -126,7 +154,7 @@ class ParquetParser(FileTypeParser):
 
         if pa.types.is_decimal(parquet_value.type):
             if parquet_format.decimal_as_float:
-                return parquet_value.as_py()
+                return float(parquet_value.as_py())
             else:
                 return str(parquet_value.as_py())
 
@@ -168,7 +196,9 @@ class ParquetParser(FileTypeParser):
         }
 
     @staticmethod
-    def parquet_type_to_schema_type(parquet_type: pa.DataType, parquet_format: ParquetFormat) -> Mapping[str, str]:
+    def parquet_type_to_schema_type(
+        parquet_type: pa.DataType, parquet_format: ParquetFormat
+    ) -> Mapping[str, str]:
         """
         Convert a pyarrow data type to an Airbyte schema type.
         Parquet data types are defined at https://arrow.apache.org/docs/python/api/datatypes.html
@@ -198,7 +228,9 @@ class ParquetParser(FileTypeParser):
     @staticmethod
     def _is_binary(parquet_type: pa.DataType) -> bool:
         return bool(
-            pa.types.is_binary(parquet_type) or pa.types.is_large_binary(parquet_type) or pa.types.is_fixed_size_binary(parquet_type)
+            pa.types.is_binary(parquet_type)
+            or pa.types.is_large_binary(parquet_type)
+            or pa.types.is_fixed_size_binary(parquet_type)
         )
 
     @staticmethod
@@ -221,13 +253,23 @@ class ParquetParser(FileTypeParser):
                 pa.types.is_time(parquet_type)
                 or pa.types.is_string(parquet_type)
                 or pa.types.is_large_string(parquet_type)
-                or ParquetParser._is_binary(parquet_type)  # Best we can do is return as a string since we do not support binary
+                or ParquetParser._is_binary(
+                    parquet_type
+                )  # Best we can do is return as a string since we do not support binary
             )
 
     @staticmethod
     def _is_object(parquet_type: pa.DataType) -> bool:
-        return bool(pa.types.is_dictionary(parquet_type) or pa.types.is_struct(parquet_type) or pa.types.is_map(parquet_type))
+        return bool(
+            pa.types.is_dictionary(parquet_type)
+            or pa.types.is_struct(parquet_type)
+            or pa.types.is_map(parquet_type)
+        )
 
     @staticmethod
     def _is_list(parquet_type: pa.DataType) -> bool:
-        return bool(pa.types.is_list(parquet_type) or pa.types.is_large_list(parquet_type) or parquet_type == pa.month_day_nano_interval())
+        return bool(
+            pa.types.is_list(parquet_type)
+            or pa.types.is_large_list(parquet_type)
+            or parquet_type == pa.month_day_nano_interval()
+        )

airbyte_cdk/sources/file_based/file_types/unstructured_parser.py

@@ -2,14 +2,24 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 import logging
+import os
 import traceback
 from datetime import datetime
 from io import BytesIO, IOBase
 from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union
 
 import backoff
-import dpath.util
+import dpath
+import nltk
 import requests
+from unstructured.file_utils.filetype import (
+    EXT_TO_FILETYPE,
+    FILETYPE_TO_MIMETYPE,
+    STR_TO_FILETYPE,
+    FileType,
+    detect_filetype,
+)
+
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.config.unstructured_format import (
@@ -19,17 +29,31 @@ from airbyte_cdk.sources.file_based.config.unstructured_format import (
     UnstructuredFormat,
 )
 from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
+from airbyte_cdk.sources.file_based.file_based_stream_reader import (
+    AbstractFileBasedStreamReader,
+    FileReadMode,
+)
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.schema_helpers import SchemaType
 from airbyte_cdk.utils import is_cloud_environment
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from unstructured.file_utils.filetype import FILETYPE_TO_MIMETYPE, STR_TO_FILETYPE, FileType, detect_filetype
 
 unstructured_partition_pdf = None
 unstructured_partition_docx = None
 unstructured_partition_pptx = None
+nltk_data_dir = "/tmp/nltk_data"
+
+try:
+    os.makedirs(nltk_data_dir, exist_ok=True)
+    nltk.data.path.append(nltk_data_dir)
+    nltk.data.find("tokenizers/punkt.zip")
+    nltk.data.find("tokenizers/punkt_tab.zip")
+    nltk.data.find("tokenizers/averaged_perceptron_tagger_eng.zip")
+except LookupError:
+    nltk.download("punkt", download_dir=nltk_data_dir)
+    nltk.download("punkt_tab", download_dir=nltk_data_dir)
+    nltk.download("averaged_perceptron_tagger_eng", download_dir=nltk_data_dir)
 
 
 def optional_decode(contents: Union[str, bytes]) -> str:
@@ -100,16 +124,21 @@ class UnstructuredParser(FileTypeParser):
         format = _extract_format(config)
         with stream_reader.open_file(file, self.file_read_mode, None, logger) as file_handle:
             filetype = self._get_filetype(file_handle, file)
-
             if filetype not in self._supported_file_types() and not format.skip_unprocessable_files:
-                raise self._create_parse_error(file, self._get_file_type_error_message(filetype))
+                raise self._create_parse_error(
+                    file,
+                    self._get_file_type_error_message(filetype),
+                )
 
             return {
                 "content": {
                     "type": "string",
                     "description": "Content of the file as markdown. Might be null if the file could not be parsed",
                 },
-                "document_key": {"type": "string", "description": "Unique identifier of the document, e.g. the file path"},
+                "document_key": {
+                    "type": "string",
+                    "description": "Unique identifier of the document, e.g. the file path",
+                },
                 "_ab_source_file_parse_error": {
                     "type": "string",
                     "description": "Error message if the file could not be parsed even though the file is supported",
@@ -148,26 +177,54 @@ class UnstructuredParser(FileTypeParser):
                     logger.warn(f"File {file.uri} cannot be parsed. Skipping it.")
                 else:
                     raise e
+            except Exception as e:
+                exception_str = str(e)
+                logger.error(f"File {file.uri} caused an error during parsing: {exception_str}.")
+                raise e
 
-    def _read_file(self, file_handle: IOBase, remote_file: RemoteFile, format: UnstructuredFormat, logger: logging.Logger) -> str:
+    def _read_file(
+        self,
+        file_handle: IOBase,
+        remote_file: RemoteFile,
+        format: UnstructuredFormat,
+        logger: logging.Logger,
+    ) -> str:
         _import_unstructured()
-        if (not unstructured_partition_pdf) or (not unstructured_partition_docx) or (not unstructured_partition_pptx):
+        if (
+            (not unstructured_partition_pdf)
+            or (not unstructured_partition_docx)
+            or (not unstructured_partition_pptx)
+        ):
             # check whether unstructured library is actually available for better error message and to ensure proper typing (can't be None after this point)
             raise Exception("unstructured library is not available")
 
-        filetype = self._get_filetype(file_handle, remote_file)
+        filetype: FileType | None = self._get_filetype(file_handle, remote_file)
 
-        if filetype == FileType.MD or filetype == FileType.TXT:
+        if filetype is None or filetype not in self._supported_file_types():
+            raise self._create_parse_error(
+                remote_file,
+                self._get_file_type_error_message(filetype),
+            )
+        if filetype in {FileType.MD, FileType.TXT}:
             file_content: bytes = file_handle.read()
             decoded_content: str = optional_decode(file_content)
             return decoded_content
-        if filetype not in self._supported_file_types():
-            raise self._create_parse_error(remote_file, self._get_file_type_error_message(filetype))
         if format.processing.mode == "local":
-            return self._read_file_locally(file_handle, filetype, format.strategy, remote_file)
+            return self._read_file_locally(
+                file_handle,
+                filetype,
+                format.strategy,
+                remote_file,
+            )
         elif format.processing.mode == "api":
             try:
-                result: str = self._read_file_remotely_with_retries(file_handle, format.processing, filetype, format.strategy, remote_file)
+                result: str = self._read_file_remotely_with_retries(
+                    file_handle,
+                    format.processing,
+                    filetype,
+                    format.strategy,
+                    remote_file,
+                )
             except Exception as e:
                 # If a parser error happens during remotely processing the file, this means the file is corrupted. This case is handled by the parse_records method, so just rethrow.
                 #
@@ -175,11 +232,15 @@ class UnstructuredParser(FileTypeParser):
                 # Once this parser leaves experimental stage, we should consider making this a system error instead for issues that might be transient.
                 if isinstance(e, RecordParseError):
                     raise e
-                raise AirbyteTracedException.from_exception(e, failure_type=FailureType.config_error)
+                raise AirbyteTracedException.from_exception(
+                    e, failure_type=FailureType.config_error
+                )
 
             return result
 
-    def _params_to_dict(self, params: Optional[List[APIParameterConfigModel]], strategy: str) -> Dict[str, Union[str, List[str]]]:
+    def _params_to_dict(
+        self, params: Optional[List[APIParameterConfigModel]], strategy: str
+    ) -> Dict[str, Union[str, List[str]]]:
         result_dict: Dict[str, Union[str, List[str]]] = {"strategy": strategy}
         if params is None:
             return result_dict
@@ -229,9 +290,16 @@ class UnstructuredParser(FileTypeParser):
 
         return True, None
 
-    @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=5, giveup=user_error)
+    @backoff.on_exception(
+        backoff.expo, requests.exceptions.RequestException, max_tries=5, giveup=user_error
+    )
     def _read_file_remotely_with_retries(
-        self, file_handle: IOBase, format: APIProcessingConfigModel, filetype: FileType, strategy: str, remote_file: RemoteFile
+        self,
+        file_handle: IOBase,
+        format: APIProcessingConfigModel,
+        filetype: FileType,
+        strategy: str,
+        remote_file: RemoteFile,
     ) -> str:
         """
         Read a file remotely, retrying up to 5 times if the error is not caused by user error. This is useful for transient network errors or the API server being overloaded temporarily.
@@ -239,7 +307,12 @@ class UnstructuredParser(FileTypeParser):
         return self._read_file_remotely(file_handle, format, filetype, strategy, remote_file)
 
     def _read_file_remotely(
-        self, file_handle: IOBase, format: APIProcessingConfigModel, filetype: FileType, strategy: str, remote_file: RemoteFile
+        self,
+        file_handle: IOBase,
+        format: APIProcessingConfigModel,
+        filetype: FileType,
+        strategy: str,
+        remote_file: RemoteFile,
     ) -> str:
         headers = {"accept": "application/json", "unstructured-api-key": format.api_key}
 
@@ -247,7 +320,9 @@ class UnstructuredParser(FileTypeParser):
 
         file_data = {"files": ("filename", file_handle, FILETYPE_TO_MIMETYPE[filetype])}
 
-        response = requests.post(f"{format.api_url}/general/v0/general", headers=headers, data=data, files=file_data)
+        response = requests.post(
+            f"{format.api_url}/general/v0/general", headers=headers, data=data, files=file_data
+        )
 
         if response.status_code == 422:
             # 422 means the file couldn't be processed, but the API is working. Treat this as a parsing error (passing an error record to the destination).
@@ -260,9 +335,15 @@ class UnstructuredParser(FileTypeParser):
 
         return self._render_markdown(json_response)
 
-    def _read_file_locally(self, file_handle: IOBase, filetype: FileType, strategy: str, remote_file: RemoteFile) -> str:
+    def _read_file_locally(
+        self, file_handle: IOBase, filetype: FileType, strategy: str, remote_file: RemoteFile
+    ) -> str:
         _import_unstructured()
-        if (not unstructured_partition_pdf) or (not unstructured_partition_docx) or (not unstructured_partition_pptx):
+        if (
+            (not unstructured_partition_pdf)
+            or (not unstructured_partition_docx)
+            or (not unstructured_partition_pptx)
+        ):
             # check whether unstructured library is actually available for better error message and to ensure proper typing (can't be None after this point)
             raise Exception("unstructured library is not available")
 
@@ -289,8 +370,14 @@ class UnstructuredParser(FileTypeParser):
 
         return self._render_markdown([element.to_dict() for element in elements])
 
-    def _create_parse_error(self, remote_file: RemoteFile, message: str) -> RecordParseError:
-        return RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=remote_file.uri, message=message)
+    def _create_parse_error(
+        self,
+        remote_file: RemoteFile,
+        message: str,
+    ) -> RecordParseError:
+        return RecordParseError(
+            FileBasedSourceError.ERROR_PARSING_RECORD, filename=remote_file.uri, message=message
+        )
 
     def _get_filetype(self, file: IOBase, remote_file: RemoteFile) -> Optional[FileType]:
         """
@@ -311,39 +398,58 @@ class UnstructuredParser(FileTypeParser):
         # detect_filetype is either using the file name or file content
         # if possible, try to leverage the file name to detect the file type
         # if the file name is not available, use the file content
-        file_type = detect_filetype(
-            filename=remote_file.uri,
-        )
-        if file_type is not None and not file_type == FileType.UNK:
+        file_type: FileType | None = None
+        try:
+            file_type = detect_filetype(
+                filename=remote_file.uri,
+            )
+        except Exception:
+            # Path doesn't exist locally. Try something else...
+            pass
+
+        if file_type and file_type != FileType.UNK:
             return file_type
 
         type_based_on_content = detect_filetype(file=file)
+        file.seek(0)  # detect_filetype is reading to read the file content, so we need to reset
 
-        # detect_filetype is reading to read the file content
-        file.seek(0)
+        if type_based_on_content and type_based_on_content != FileType.UNK:
+            return type_based_on_content
 
-        return type_based_on_content
+        extension = "." + remote_file.uri.split(".")[-1].lower()
+        if extension in EXT_TO_FILETYPE:
+            return EXT_TO_FILETYPE[extension]
+
+        return None
 
     def _supported_file_types(self) -> List[Any]:
         return [FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX, FileType.TXT]
 
-    def _get_file_type_error_message(self, file_type: FileType) -> str:
+    def _get_file_type_error_message(
+        self,
+        file_type: FileType | None,
+    ) -> str:
         supported_file_types = ", ".join([str(type) for type in self._supported_file_types()])
-        return f"File type {file_type} is not supported. Supported file types are {supported_file_types}"
+        return f"File type {file_type or 'None'!s} is not supported. Supported file types are {supported_file_types}"
 
     def _render_markdown(self, elements: List[Any]) -> str:
         return "\n\n".join((self._convert_to_markdown(el) for el in elements))
 
     def _convert_to_markdown(self, el: Dict[str, Any]) -> str:
-        if dpath.util.get(el, "type") == "Title":
-            heading_str = "#" * (dpath.util.get(el, "metadata/category_depth", default=1) or 1)
-            return f"{heading_str} {dpath.util.get(el, 'text')}"
-        elif dpath.util.get(el, "type") == "ListItem":
-            return f"- {dpath.util.get(el, 'text')}"
-        elif dpath.util.get(el, "type") == "Formula":
-            return f"```\n{dpath.util.get(el, 'text')}\n```"
+        if dpath.get(el, "type") == "Title":
+            category_depth = dpath.get(el, "metadata/category_depth", default=1) or 1
+            if not isinstance(category_depth, int):
+                category_depth = (
+                    int(category_depth) if isinstance(category_depth, (str, float)) else 1
+                )
+            heading_str = "#" * category_depth
+            return f"{heading_str} {dpath.get(el, 'text')}"
+        elif dpath.get(el, "type") == "ListItem":
+            return f"- {dpath.get(el, 'text')}"
+        elif dpath.get(el, "type") == "Formula":
+            return f"```\n{dpath.get(el, 'text')}\n```"
         else:
-            return str(dpath.util.get(el, "text", default=""))
+            return str(dpath.get(el, "text", default=""))
 
     @property
     def file_read_mode(self) -> FileReadMode:

airbyte_cdk/sources/file_based/remote_file.py

@@ -5,7 +5,7 @@
 from datetime import datetime
 from typing import Optional
 
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 
 class RemoteFile(BaseModel):