airbyte-cdk 6.20.2.dev0__py3-none-any.whl → 6.21.0__py3-none-any.whl

airbyte_cdk/sources/file_based/exceptions.py

@@ -111,6 +111,40 @@ class ErrorListingFiles(BaseFileBasedSourceError):
     pass
 
 
+class DuplicatedFilesError(BaseFileBasedSourceError):
+    def __init__(self, duplicated_files_names: List[dict[str, List[str]]], **kwargs: Any):
+        self._duplicated_files_names = duplicated_files_names
+        self._stream_name: str = kwargs["stream"]
+        super().__init__(self._format_duplicate_files_error_message(), **kwargs)
+
+    def _format_duplicate_files_error_message(self) -> str:
+        duplicated_files_messages = []
+        for duplicated_file in self._duplicated_files_names:
+            for duplicated_file_name, file_paths in duplicated_file.items():
+                file_duplicated_message = (
+                    f"{len(file_paths)} duplicates found for file name {duplicated_file_name}:\n\n"
+                    + "".join(f"\n - {file_paths}")
+                )
+                duplicated_files_messages.append(file_duplicated_message)
+
+        error_message = (
+            f"ERROR: Duplicate filenames found for stream {self._stream_name}. "
+            "Duplicate file names are not allowed if the Preserve Sub-Directories in File Paths option is disabled. "
+            "Please remove or rename the duplicate files before attempting to re-run the sync.\n\n"
+            + "\n".join(duplicated_files_messages)
+        )
+
+        return error_message
+
+    def __repr__(self) -> str:
+        """Return a string representation of the exception."""
+        class_name = self.__class__.__name__
+        properties_str = ", ".join(
+            f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_")
+        )
+        return f"{class_name}({properties_str})"
+
+
 class CustomFileBasedException(AirbyteTracedException):
     """
     A specialized exception for file-based connectors.

airbyte_cdk/sources/file_based/file_based_source.py

@@ -242,7 +242,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         stream=self._make_default_stream(
                             stream_config=stream_config,
                             cursor=cursor,
-                            use_file_transfer=self._use_file_transfer(parsed_config),
+                            parsed_config=parsed_config,
                         ),
                         source=self,
                         logger=self.logger,
@@ -273,7 +273,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         stream=self._make_default_stream(
                             stream_config=stream_config,
                             cursor=cursor,
-                            use_file_transfer=self._use_file_transfer(parsed_config),
+                            parsed_config=parsed_config,
                         ),
                         source=self,
                         logger=self.logger,
@@ -285,7 +285,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                     stream = self._make_default_stream(
                         stream_config=stream_config,
                         cursor=cursor,
-                        use_file_transfer=self._use_file_transfer(parsed_config),
+                        parsed_config=parsed_config,
                     )
 
                 streams.append(stream)
@@ -298,7 +298,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         self,
         stream_config: FileBasedStreamConfig,
         cursor: Optional[AbstractFileBasedCursor],
-        use_file_transfer: bool = False,
+        parsed_config: AbstractFileBasedSpec,
     ) -> AbstractFileBasedStream:
         return DefaultFileBasedStream(
             config=stream_config,
@@ -310,7 +310,8 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             validation_policy=self._validate_and_get_validation_policy(stream_config),
             errors_collector=self.errors_collector,
             cursor=cursor,
-            use_file_transfer=use_file_transfer,
+            use_file_transfer=self._use_file_transfer(parsed_config),
+            preserve_directory_structure=self._preserve_directory_structure(parsed_config),
         )
 
     def _get_stream_from_catalog(
@@ -385,3 +386,25 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             and parsed_config.delivery_method.delivery_type == "use_file_transfer"
         )
         return use_file_transfer
+
+    @staticmethod
+    def _preserve_directory_structure(parsed_config: AbstractFileBasedSpec) -> bool:
+        """
+        Determines whether to preserve directory structure during file transfer.
+
+        When enabled, files maintain their subdirectory paths in the destination.
+        When disabled, files are flattened to the root of the destination.
+
+        Args:
+            parsed_config: The parsed configuration containing delivery method settings
+
+        Returns:
+            True if directory structure should be preserved (default), False otherwise
+        """
+        if (
+            FileBasedSource._use_file_transfer(parsed_config)
+            and hasattr(parsed_config.delivery_method, "preserve_directory_structure")
+            and parsed_config.delivery_method.preserve_directory_structure is not None
+        ):
+            return parsed_config.delivery_method.preserve_directory_structure
+        return True

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -135,6 +135,17 @@ class AbstractFileBasedStreamReader(ABC):
             return use_file_transfer
         return False
 
+    def preserve_directory_structure(self) -> bool:
+        # fall back to preserve subdirectories if config is not present or incomplete
+        if (
+            self.use_file_transfer()
+            and self.config
+            and hasattr(self.config.delivery_method, "preserve_directory_structure")
+            and self.config.delivery_method.preserve_directory_structure is not None
+        ):
+            return self.config.delivery_method.preserve_directory_structure
+        return True
+
     @abstractmethod
     def get_file(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
@@ -159,10 +170,13 @@ class AbstractFileBasedStreamReader(ABC):
         """
         ...
 
-    @staticmethod
-    def _get_file_transfer_paths(file: RemoteFile, local_directory: str) -> List[str]:
-        # Remove left slashes from source path format to make relative path for writing locally
-        file_relative_path = file.uri.lstrip("/")
+    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
+        preserve_directory_structure = self.preserve_directory_structure()
+        if preserve_directory_structure:
+            # Remove left slashes from source path format to make relative path for writing locally
+            file_relative_path = file.uri.lstrip("/")
+        else:
+            file_relative_path = path.basename(file.uri)
         local_file_path = path.join(local_directory, file_relative_path)
 
         # Ensure the local directory exists

airbyte_cdk/sources/file_based/file_types/unstructured_parser.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 import logging
+import os
 import traceback
 from datetime import datetime
 from io import BytesIO, IOBase
@@ -42,12 +43,34 @@ unstructured_partition_pdf = None
 unstructured_partition_docx = None
 unstructured_partition_pptx = None
 
+AIRBYTE_NLTK_DATA_DIR = "/airbyte/nltk_data"
+TMP_NLTK_DATA_DIR = "/tmp/nltk_data"
+
+
+def get_nltk_temp_folder() -> str:
+    """
+    For non-root connectors /tmp is not currently writable, but we should allow it in the future.
+    It's safe to use /airbyte for now. Fallback to /tmp for local development.
+    """
+    try:
+        nltk_data_dir = AIRBYTE_NLTK_DATA_DIR
+        os.makedirs(nltk_data_dir, exist_ok=True)
+    except OSError:
+        nltk_data_dir = TMP_NLTK_DATA_DIR
+        os.makedirs(nltk_data_dir, exist_ok=True)
+    return nltk_data_dir
+
+
 try:
+    nltk_data_dir = get_nltk_temp_folder()
+    nltk.data.path.append(nltk_data_dir)
     nltk.data.find("tokenizers/punkt.zip")
     nltk.data.find("tokenizers/punkt_tab.zip")
+    nltk.data.find("tokenizers/averaged_perceptron_tagger_eng.zip")
 except LookupError:
-    nltk.download("punkt")
-    nltk.download("punkt_tab")
+    nltk.download("punkt", download_dir=nltk_data_dir, quiet=True)
+    nltk.download("punkt_tab", download_dir=nltk_data_dir, quiet=True)
+    nltk.download("averaged_perceptron_tagger_eng", download_dir=nltk_data_dir, quiet=True)
 
 
 def optional_decode(contents: Union[str, bytes]) -> str:

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -5,14 +5,17 @@
 import asyncio
 import itertools
 import traceback
+from collections import defaultdict
 from copy import deepcopy
 from functools import cache
-from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Union
+from os import path
+from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.exceptions import (
+    DuplicatedFilesError,
     FileBasedSourceError,
     InvalidSchemaError,
     MissingSchemaError,
@@ -43,6 +46,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     """
 
     FILE_TRANSFER_KW = "use_file_transfer"
+    PRESERVE_DIRECTORY_STRUCTURE_KW = "preserve_directory_structure"
+    FILES_KEY = "files"
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
     ab_last_mod_col = "_ab_source_file_last_modified"
     ab_file_name_col = "_ab_source_file_url"
@@ -50,10 +55,15 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     source_file_url = "source_file_url"
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
     use_file_transfer = False
+    preserve_directory_structure = True
 
     def __init__(self, **kwargs: Any):
         if self.FILE_TRANSFER_KW in kwargs:
             self.use_file_transfer = kwargs.pop(self.FILE_TRANSFER_KW, False)
+        if self.PRESERVE_DIRECTORY_STRUCTURE_KW in kwargs:
+            self.preserve_directory_structure = kwargs.pop(
+                self.PRESERVE_DIRECTORY_STRUCTURE_KW, True
+            )
         super().__init__(**kwargs)
 
     @property
@@ -98,15 +108,33 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
+    def _duplicated_files_names(
+        self, slices: List[dict[str, List[RemoteFile]]]
+    ) -> List[dict[str, List[str]]]:
+        seen_file_names: Dict[str, List[str]] = defaultdict(list)
+        for file_slice in slices:
+            for file_found in file_slice[self.FILES_KEY]:
+                file_name = path.basename(file_found.uri)
+                seen_file_names[file_name].append(file_found.uri)
+        return [
+            {file_name: paths} for file_name, paths in seen_file_names.items() if len(paths) > 1
+        ]
+
     def compute_slices(self) -> Iterable[Optional[Mapping[str, Any]]]:
         # Sort files by last_modified, uri and return them grouped by last_modified
         all_files = self.list_files()
         files_to_read = self._cursor.get_files_to_sync(all_files, self.logger)
         sorted_files_to_read = sorted(files_to_read, key=lambda f: (f.last_modified, f.uri))
         slices = [
-            {"files": list(group[1])}
+            {self.FILES_KEY: list(group[1])}
             for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)
         ]
+        if slices and not self.preserve_directory_structure:
+            duplicated_files_names = self._duplicated_files_names(slices)
+            if duplicated_files_names:
+                raise DuplicatedFilesError(
+                    stream=self.name, duplicated_files_names=duplicated_files_names
+                )
         return slices
 
     def transform_record(

airbyte_cdk/sources/streams/concurrent/cursor.py

@@ -196,9 +196,7 @@ class ConcurrentCursor(Cursor):
 
     @property
     def state(self) -> MutableMapping[str, Any]:
-        return self._connector_state_converter.convert_to_state_message(
-            self.cursor_field, self._concurrent_state
-        )
+        return self._concurrent_state
 
     @property
     def cursor_field(self) -> CursorField:
@@ -243,10 +241,10 @@ class ConcurrentCursor(Cursor):
         return self._connector_state_converter.parse_value(self._cursor_field.extract_value(record))
 
     def close_partition(self, partition: Partition) -> None:
-        slice_count_before = len(self._concurrent_state.get("slices", []))
+        slice_count_before = len(self.state.get("slices", []))
         self._add_slice_to_state(partition)
         if slice_count_before < len(
-            self._concurrent_state["slices"]
+            self.state["slices"]
         ):  # only emit if at least one slice has been processed
             self._merge_partitions()
             self._emit_state_message()
@@ -258,11 +256,11 @@ class ConcurrentCursor(Cursor):
         )
 
         if self._slice_boundary_fields:
-            if "slices" not in self._concurrent_state:
+            if "slices" not in self.state:
                 raise RuntimeError(
                     f"The state for stream {self._stream_name} should have at least one slice to delineate the sync start time, but no slices are present. This is unexpected. Please contact Support."
                 )
-            self._concurrent_state["slices"].append(
+            self.state["slices"].append(
                 {
                     self._connector_state_converter.START_KEY: self._extract_from_slice(
                         partition, self._slice_boundary_fields[self._START_BOUNDARY]
@@ -290,7 +288,7 @@ class ConcurrentCursor(Cursor):
                     "expected. Please contact the Airbyte team."
                 )
 
-            self._concurrent_state["slices"].append(
+            self.state["slices"].append(
                 {
                     self._connector_state_converter.START_KEY: self.start,
                     self._connector_state_converter.END_KEY: most_recent_cursor_value,
@@ -302,7 +300,9 @@ class ConcurrentCursor(Cursor):
         self._connector_state_manager.update_state_for_stream(
             self._stream_name,
             self._stream_namespace,
-            self.state,
+            self._connector_state_converter.convert_to_state_message(
+                self._cursor_field, self.state
+            ),
         )
         state_message = self._connector_state_manager.create_state_message(
             self._stream_name, self._stream_namespace
@@ -310,9 +310,7 @@ class ConcurrentCursor(Cursor):
         self._message_repository.emit_message(state_message)
 
     def _merge_partitions(self) -> None:
-        self._concurrent_state["slices"] = self._connector_state_converter.merge_intervals(
-            self._concurrent_state["slices"]
-        )
+        self.state["slices"] = self._connector_state_converter.merge_intervals(self.state["slices"])
 
     def _extract_from_slice(self, partition: Partition, key: str) -> CursorValueType:
         try:
@@ -349,42 +347,36 @@ class ConcurrentCursor(Cursor):
         if self._start is not None and self._is_start_before_first_slice():
             yield from self._split_per_slice_range(
                 self._start,
-                self._concurrent_state["slices"][0][self._connector_state_converter.START_KEY],
+                self.state["slices"][0][self._connector_state_converter.START_KEY],
                 False,
             )
 
-        if len(self._concurrent_state["slices"]) == 1:
+        if len(self.state["slices"]) == 1:
             yield from self._split_per_slice_range(
                 self._calculate_lower_boundary_of_last_slice(
-                    self._concurrent_state["slices"][0][self._connector_state_converter.END_KEY]
+                    self.state["slices"][0][self._connector_state_converter.END_KEY]
                 ),
                 self._end_provider(),
                 True,
             )
-        elif len(self._concurrent_state["slices"]) > 1:
-            for i in range(len(self._concurrent_state["slices"]) - 1):
+        elif len(self.state["slices"]) > 1:
+            for i in range(len(self.state["slices"]) - 1):
                 if self._cursor_granularity:
                     yield from self._split_per_slice_range(
-                        self._concurrent_state["slices"][i][self._connector_state_converter.END_KEY]
+                        self.state["slices"][i][self._connector_state_converter.END_KEY]
                         + self._cursor_granularity,
-                        self._concurrent_state["slices"][i + 1][
-                            self._connector_state_converter.START_KEY
-                        ],
+                        self.state["slices"][i + 1][self._connector_state_converter.START_KEY],
                         False,
                     )
                 else:
                     yield from self._split_per_slice_range(
-                        self._concurrent_state["slices"][i][
-                            self._connector_state_converter.END_KEY
-                        ],
-                        self._concurrent_state["slices"][i + 1][
-                            self._connector_state_converter.START_KEY
-                        ],
+                        self.state["slices"][i][self._connector_state_converter.END_KEY],
+                        self.state["slices"][i + 1][self._connector_state_converter.START_KEY],
                         False,
                     )
             yield from self._split_per_slice_range(
                 self._calculate_lower_boundary_of_last_slice(
-                    self._concurrent_state["slices"][-1][self._connector_state_converter.END_KEY]
+                    self.state["slices"][-1][self._connector_state_converter.END_KEY]
                 ),
                 self._end_provider(),
                 True,
@@ -395,8 +387,7 @@ class ConcurrentCursor(Cursor):
     def _is_start_before_first_slice(self) -> bool:
         return (
             self._start is not None
-            and self._start
-            < self._concurrent_state["slices"][0][self._connector_state_converter.START_KEY]
+            and self._start < self.state["slices"][0][self._connector_state_converter.START_KEY]
         )
 
     def _calculate_lower_boundary_of_last_slice(

airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -81,10 +81,10 @@ class AbstractOauth2Authenticator(AuthBase):
         Override to define additional parameters
         """
         payload: MutableMapping[str, Any] = {
-            "grant_type": self.get_grant_type(),
-            "client_id": self.get_client_id(),
-            "client_secret": self.get_client_secret(),
-            "refresh_token": self.get_refresh_token(),
+            self.get_grant_type_name(): self.get_grant_type(),
+            self.get_client_id_name(): self.get_client_id(),
+            self.get_client_secret_name(): self.get_client_secret(),
+            self.get_refresh_token_name(): self.get_refresh_token(),
         }
 
         if self.get_scopes():
@@ -98,6 +98,14 @@ class AbstractOauth2Authenticator(AuthBase):
 
         return payload
 
+    def build_refresh_request_headers(self) -> Mapping[str, Any] | None:
+        """
+        Returns the request headers to set on the refresh request
+
+        """
+        headers = self.get_refresh_request_headers()
+        return headers if headers else None
+
     def _wrap_refresh_token_exception(
         self, exception: requests.exceptions.RequestException
     ) -> bool:
@@ -128,6 +136,7 @@ class AbstractOauth2Authenticator(AuthBase):
                 method="POST",
                 url=self.get_token_refresh_endpoint(),  # type: ignore # returns None, if not provided, but str | bytes is expected.
                 data=self.build_refresh_request_body(),
+                headers=self.build_refresh_request_headers(),
             )
             if response.ok:
                 response_json = response.json()
@@ -206,14 +215,26 @@ class AbstractOauth2Authenticator(AuthBase):
     def get_token_refresh_endpoint(self) -> Optional[str]:
         """Returns the endpoint to refresh the access token"""
 
+    @abstractmethod
+    def get_client_id_name(self) -> str:
+        """The client id name to authenticate"""
+
     @abstractmethod
     def get_client_id(self) -> str:
         """The client id to authenticate"""
 
+    @abstractmethod
+    def get_client_secret_name(self) -> str:
+        """The client secret name to authenticate"""
+
     @abstractmethod
     def get_client_secret(self) -> str:
         """The client secret to authenticate"""
 
+    @abstractmethod
+    def get_refresh_token_name(self) -> str:
+        """The refresh token name to authenticate"""
+
     @abstractmethod
     def get_refresh_token(self) -> Optional[str]:
         """The token used to refresh the access token when it expires"""
@@ -242,10 +263,18 @@ class AbstractOauth2Authenticator(AuthBase):
     def get_refresh_request_body(self) -> Mapping[str, Any]:
         """Returns the request body to set on the refresh request"""
 
+    @abstractmethod
+    def get_refresh_request_headers(self) -> Mapping[str, Any]:
+        """Returns the request headers to set on the refresh request"""
+
     @abstractmethod
     def get_grant_type(self) -> str:
         """Returns grant_type specified for requesting access_token"""
 
+    @abstractmethod
+    def get_grant_type_name(self) -> str:
+        """Returns grant_type specified name for requesting access_token"""
+
     @property
     @abstractmethod
     def access_token(self) -> str:

airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py

@@ -30,12 +30,17 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
         client_id: str,
         client_secret: str,
         refresh_token: str,
+        client_id_name: str = "client_id",
+        client_secret_name: str = "client_secret",
+        refresh_token_name: str = "refresh_token",
         scopes: List[str] | None = None,
         token_expiry_date: pendulum.DateTime | None = None,
         token_expiry_date_format: str | None = None,
         access_token_name: str = "access_token",
         expires_in_name: str = "expires_in",
         refresh_request_body: Mapping[str, Any] | None = None,
+        refresh_request_headers: Mapping[str, Any] | None = None,
+        grant_type_name: str = "grant_type",
         grant_type: str = "refresh_token",
         token_expiry_is_time_of_expiration: bool = False,
         refresh_token_error_status_codes: Tuple[int, ...] = (),
@@ -43,13 +48,18 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
         refresh_token_error_values: Tuple[str, ...] = (),
     ):
         self._token_refresh_endpoint = token_refresh_endpoint
+        self._client_secret_name = client_secret_name
         self._client_secret = client_secret
+        self._client_id_name = client_id_name
         self._client_id = client_id
+        self._refresh_token_name = refresh_token_name
         self._refresh_token = refresh_token
         self._scopes = scopes
         self._access_token_name = access_token_name
         self._expires_in_name = expires_in_name
         self._refresh_request_body = refresh_request_body
+        self._refresh_request_headers = refresh_request_headers
+        self._grant_type_name = grant_type_name
         self._grant_type = grant_type
 
         self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1)  # type: ignore [no-untyped-call]
@@ -63,12 +73,21 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
     def get_token_refresh_endpoint(self) -> str:
         return self._token_refresh_endpoint
 
+    def get_client_id_name(self) -> str:
+        return self._client_id_name
+
     def get_client_id(self) -> str:
         return self._client_id
 
+    def get_client_secret_name(self) -> str:
+        return self._client_secret_name
+
     def get_client_secret(self) -> str:
         return self._client_secret
 
+    def get_refresh_token_name(self) -> str:
+        return self._refresh_token_name
+
     def get_refresh_token(self) -> str:
         return self._refresh_token
 
@@ -84,6 +103,12 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
     def get_refresh_request_body(self) -> Mapping[str, Any]:
         return self._refresh_request_body  # type: ignore [return-value]
 
+    def get_refresh_request_headers(self) -> Mapping[str, Any]:
+        return self._refresh_request_headers  # type: ignore [return-value]
+
+    def get_grant_type_name(self) -> str:
+        return self._grant_type_name
+
     def get_grant_type(self) -> str:
         return self._grant_type
 
@@ -129,8 +154,12 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         expires_in_name: str = "expires_in",
         refresh_token_name: str = "refresh_token",
         refresh_request_body: Mapping[str, Any] | None = None,
+        refresh_request_headers: Mapping[str, Any] | None = None,
+        grant_type_name: str = "grant_type",
         grant_type: str = "refresh_token",
+        client_id_name: str = "client_id",
         client_id: Optional[str] = None,
+        client_secret_name: str = "client_secret",
         client_secret: Optional[str] = None,
         access_token_config_path: Sequence[str] = ("credentials", "access_token"),
         refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"),
@@ -151,6 +180,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             expires_in_name (str, optional): Name of the name of the field that characterizes when the current access token will expire, used to parse the refresh token response. Defaults to "expires_in".
             refresh_token_name (str, optional): Name of the name of the refresh token field, used to parse the refresh token response. Defaults to "refresh_token".
             refresh_request_body (Mapping[str, Any], optional): Custom key value pair that will be added to the refresh token request body. Defaults to None.
+            refresh_request_headers (Mapping[str, Any], optional): Custom key value pair that will be added to the refresh token request headers. Defaults to None.
             grant_type (str, optional): OAuth grant type. Defaults to "refresh_token".
             client_id (Optional[str]): The client id to authenticate. If not specified, defaults to credentials.client_id in the config object.
             client_secret (Optional[str]): The client secret to authenticate. If not specified, defaults to credentials.client_secret in the config object.
@@ -174,23 +204,31 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
                 ("credentials", "client_secret"),
             )
         )
+        self._client_id_name = client_id_name
+        self._client_secret_name = client_secret_name
         self._access_token_config_path = access_token_config_path
         self._refresh_token_config_path = refresh_token_config_path
         self._token_expiry_date_config_path = token_expiry_date_config_path
         self._token_expiry_date_format = token_expiry_date_format
         self._refresh_token_name = refresh_token_name
+        self._grant_type_name = grant_type_name
         self._connector_config = connector_config
         self.__message_repository = message_repository
         super().__init__(
-            token_refresh_endpoint,
-            self.get_client_id(),
-            self.get_client_secret(),
-            self.get_refresh_token(),
+            token_refresh_endpoint=token_refresh_endpoint,
+            client_id_name=self._client_id_name,
+            client_id=self.get_client_id(),
+            client_secret_name=self._client_secret_name,
+            client_secret=self.get_client_secret(),
+            refresh_token=self.get_refresh_token(),
+            refresh_token_name=self._refresh_token_name,
             scopes=scopes,
             token_expiry_date=self.get_token_expiry_date(),
             access_token_name=access_token_name,
             expires_in_name=expires_in_name,
             refresh_request_body=refresh_request_body,
+            refresh_request_headers=refresh_request_headers,
+            grant_type_name=self._grant_type_name,
             grant_type=grant_type,
             token_expiry_date_format=token_expiry_date_format,
             token_expiry_is_time_of_expiration=token_expiry_is_time_of_expiration,

airbyte_cdk/sources/types.py

@@ -152,3 +152,6 @@ class StreamSlice(Mapping[str, Any]):
 
     def __hash__(self) -> int:
         return hash(orjson.dumps(self._stream_slice, option=orjson.OPT_SORT_KEYS))
+
+    def __bool__(self) -> bool:
+        return bool(self._stream_slice) or bool(self._extra_fields)

airbyte_cdk/sources/utils/transform.py

@@ -9,6 +9,7 @@ from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
 
 from jsonschema import Draft7Validator, RefResolver, ValidationError, Validator, validators
 
+MAX_NESTING_DEPTH = 3
 json_to_python_simple = {
     "string": str,
     "number": float,
@@ -225,6 +226,31 @@ class TypeTransformer:
             logger.warning(self.get_error_message(e))
 
     def get_error_message(self, e: ValidationError) -> str:
-        instance_json_type = python_to_json[type(e.instance)]
-        key_path = "." + ".".join(map(str, e.path))
-        return f"Failed to transform value {repr(e.instance)} of type '{instance_json_type}' to '{e.validator_value}', key path: '{key_path}'"
+        """
+        Construct a sanitized error message from a ValidationError instance.
+        """
+        field_path = ".".join(map(str, e.path))
+        type_structure = self._get_type_structure(e.instance)
+
+        return f"Failed to transform value from type '{type_structure}' to type '{e.validator_value}' at path: '{field_path}'"
+
+    def _get_type_structure(self, input_data: Any, current_depth: int = 0) -> Any:
+        """
+        Get the structure of a given input data for use in error message construction.
+        """
+        # Handle null values
+        if input_data is None:
+            return "null"
+
+        # Avoid recursing too deep
+        if current_depth >= MAX_NESTING_DEPTH:
+            return "object" if isinstance(input_data, dict) else python_to_json[type(input_data)]
+
+        if isinstance(input_data, dict):
+            return {
+                key: self._get_type_structure(field_value, current_depth + 1)
+                for key, field_value in input_data.items()
+            }
+
+        else:
+            return python_to_json[type(input_data)]