airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

airbyte_cdk/sources/file_based/exceptions.py

@@ -11,26 +11,21 @@ from airbyte_cdk.utils import AirbyteTracedException
 
 class FileBasedSourceError(Enum):
     EMPTY_STREAM = "No files were identified in the stream. This may be because there are no files in the specified container, or because your glob patterns did not match any files. Please verify that your source contains files last modified after the start_date and that your glob patterns are not overly strict."
-    GLOB_PARSE_ERROR = (
-        "Error parsing glob pattern. Please refer to the glob pattern rules at https://facelessuser.github.io/wcmatch/glob/#split."
-    )
+    GLOB_PARSE_ERROR = "Error parsing glob pattern. Please refer to the glob pattern rules at https://facelessuser.github.io/wcmatch/glob/#split."
+    ENCODING_ERROR = "File encoding error. The configured encoding must match file encoding."
     ERROR_CASTING_VALUE = "Could not cast the value to the expected type."
     ERROR_CASTING_VALUE_UNRECOGNIZED_TYPE = "Could not cast the value to the expected type because the type is not recognized. Valid types are null, array, boolean, integer, number, object, and string."
     ERROR_DECODING_VALUE = "Expected a JSON-decodeable value but could not decode record."
-    ERROR_LISTING_FILES = (
-        "Error listing files. Please check the credentials provided in the config and verify that they provide permission to list files."
-    )
-    ERROR_READING_FILE = (
-        "Error opening file. Please check the credentials provided in the config and verify that they provide permission to read files."
-    )
+    ERROR_LISTING_FILES = "Error listing files. Please check the credentials provided in the config and verify that they provide permission to list files."
+    ERROR_READING_FILE = "Error opening file. Please check the credentials provided in the config and verify that they provide permission to read files."
     ERROR_PARSING_RECORD = "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable."
-    ERROR_PARSING_USER_PROVIDED_SCHEMA = "The provided schema could not be transformed into valid JSON Schema."
+    ERROR_PARSING_USER_PROVIDED_SCHEMA = (
+        "The provided schema could not be transformed into valid JSON Schema."
+    )
     ERROR_VALIDATING_RECORD = "One or more records do not pass the schema validation policy. Please modify your input schema, or select a more lenient validation policy."
     ERROR_PARSING_RECORD_MISMATCHED_COLUMNS = "A header field has resolved to `None`. This indicates that the CSV has more rows than the number of header fields. If you input your schema or headers, please verify that the number of columns corresponds to the number of columns in your CSV's rows."
     ERROR_PARSING_RECORD_MISMATCHED_ROWS = "A row's value has resolved to `None`. This indicates that the CSV has more columns in the header field than the number of columns in the row(s). If you input your schema or headers, please verify that the number of columns corresponds to the number of columns in your CSV's rows."
-    STOP_SYNC_PER_SCHEMA_VALIDATION_POLICY = (
-        "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema."
-    )
+    STOP_SYNC_PER_SCHEMA_VALIDATION_POLICY = "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema."
     NULL_VALUE_IN_SCHEMA = "Error during schema inference: no type was detected for key."
     UNRECOGNIZED_TYPE = "Error during schema inference: unrecognized type."
     SCHEMA_INFERENCE_ERROR = "Error inferring schema from files. Are the files valid?"
@@ -38,7 +33,9 @@ class FileBasedSourceError(Enum):
     CONFIG_VALIDATION_ERROR = "Error creating stream config object."
     MISSING_SCHEMA = "Expected `json_schema` in the configured catalog but it is missing."
     UNDEFINED_PARSER = "No parser is defined for this file type."
-    UNDEFINED_VALIDATION_POLICY = "The validation policy defined in the config does not exist for the source."
+    UNDEFINED_VALIDATION_POLICY = (
+        "The validation policy defined in the config does not exist for the source."
+    )
 
 
 class FileBasedErrorsCollector:
@@ -69,7 +66,9 @@ class BaseFileBasedSourceError(Exception):
     def __init__(self, error: Union[FileBasedSourceError, str], **kwargs):  # type: ignore # noqa
         if isinstance(error, FileBasedSourceError):
             error = FileBasedSourceError(error).value
-        super().__init__(f"{error} Contact Support if you need assistance.\n{' '.join([f'{k}={v}' for k, v in kwargs.items()])}")
+        super().__init__(
+            f"{error} Contact Support if you need assistance.\n{' '.join([f'{k}={v}' for k, v in kwargs.items()])}"
+        )
 
 
 class ConfigValidationError(BaseFileBasedSourceError):
@@ -112,6 +111,40 @@ class ErrorListingFiles(BaseFileBasedSourceError):
     pass
 
 
+class DuplicatedFilesError(BaseFileBasedSourceError):
+    def __init__(self, duplicated_files_names: List[dict[str, List[str]]], **kwargs: Any):
+        self._duplicated_files_names = duplicated_files_names
+        self._stream_name: str = kwargs["stream"]
+        super().__init__(self._format_duplicate_files_error_message(), **kwargs)
+
+    def _format_duplicate_files_error_message(self) -> str:
+        duplicated_files_messages = []
+        for duplicated_file in self._duplicated_files_names:
+            for duplicated_file_name, file_paths in duplicated_file.items():
+                file_duplicated_message = (
+                    f"{len(file_paths)} duplicates found for file name {duplicated_file_name}:\n\n"
+                    + "".join(f"\n - {file_paths}")
+                )
+                duplicated_files_messages.append(file_duplicated_message)
+
+        error_message = (
+            f"ERROR: Duplicate filenames found for stream {self._stream_name}. "
+            "Duplicate file names are not allowed if the Preserve Sub-Directories in File Paths option is disabled. "
+            "Please remove or rename the duplicate files before attempting to re-run the sync.\n\n"
+            + "\n".join(duplicated_files_messages)
+        )
+
+        return error_message
+
+    def __repr__(self) -> str:
+        """Return a string representation of the exception."""
+        class_name = self.__class__.__name__
+        properties_str = ", ".join(
+            f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_")
+        )
+        return f"{class_name}({properties_str})"
+
+
 class CustomFileBasedException(AirbyteTracedException):
     """
     A specialized exception for file-based connectors.
@@ -120,3 +153,7 @@ class CustomFileBasedException(AirbyteTracedException):
     """
 
     pass
+
+
+class FileSizeLimitError(CustomFileBasedException):
+    pass

airbyte_cdk/sources/file_based/file_based_source.py

@@ -6,7 +6,9 @@ import logging
 import traceback
 from abc import ABC
 from collections import Counter
-from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Type, Union
+from typing import Any, Iterator, List, Mapping, Optional, Tuple, Type, Union
+
+from pydantic.v1.error_wrappers import ValidationError
 
 from airbyte_cdk.logger import AirbyteLogFormatter, init_logger
 from airbyte_cdk.models import (
@@ -22,15 +24,31 @@ from airbyte_cdk.models import (
 from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource
 from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
-from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy, DefaultFileBasedAvailabilityStrategy
+from airbyte_cdk.sources.file_based.availability_strategy import (
+    AbstractFileBasedAvailabilityStrategy,
+    DefaultFileBasedAvailabilityStrategy,
+)
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ValidationPolicy
-from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy, DefaultDiscoveryPolicy
-from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedErrorsCollector, FileBasedSourceError
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import (
+    FileBasedStreamConfig,
+    ValidationPolicy,
+)
+from airbyte_cdk.sources.file_based.discovery_policy import (
+    AbstractDiscoveryPolicy,
+    DefaultDiscoveryPolicy,
+)
+from airbyte_cdk.sources.file_based.exceptions import (
+    ConfigValidationError,
+    FileBasedErrorsCollector,
+    FileBasedSourceError,
+)
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
 from airbyte_cdk.sources.file_based.file_types import default_parsers
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
-from airbyte_cdk.sources.file_based.schema_validation_policies import DEFAULT_SCHEMA_VALIDATION_POLICIES, AbstractSchemaValidationPolicy
+from airbyte_cdk.sources.file_based.schema_validation_policies import (
+    DEFAULT_SCHEMA_VALIDATION_POLICIES,
+    AbstractSchemaValidationPolicy,
+)
 from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream, DefaultFileBasedStream
 from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamFacade
 from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
@@ -44,7 +62,6 @@ from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
 from airbyte_cdk.utils.analytics_message import create_analytics_message
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from pydantic.error_wrappers import ValidationError
 
 DEFAULT_CONCURRENCY = 100
 MAX_CONCURRENCY = 100
@@ -61,29 +78,41 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         spec_class: Type[AbstractFileBasedSpec],
         catalog: Optional[ConfiguredAirbyteCatalog],
         config: Optional[Mapping[str, Any]],
-        state: Optional[MutableMapping[str, Any]],
+        state: Optional[List[AirbyteStateMessage]],
         availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy] = None,
         discovery_policy: AbstractDiscoveryPolicy = DefaultDiscoveryPolicy(),
         parsers: Mapping[Type[Any], FileTypeParser] = default_parsers,
-        validation_policies: Mapping[ValidationPolicy, AbstractSchemaValidationPolicy] = DEFAULT_SCHEMA_VALIDATION_POLICIES,
-        cursor_cls: Type[Union[AbstractConcurrentFileBasedCursor, AbstractFileBasedCursor]] = FileBasedConcurrentCursor,
+        validation_policies: Mapping[
+            ValidationPolicy, AbstractSchemaValidationPolicy
+        ] = DEFAULT_SCHEMA_VALIDATION_POLICIES,
+        cursor_cls: Type[
+            Union[AbstractConcurrentFileBasedCursor, AbstractFileBasedCursor]
+        ] = FileBasedConcurrentCursor,
     ):
         self.stream_reader = stream_reader
         self.spec_class = spec_class
         self.config = config
         self.catalog = catalog
         self.state = state
-        self.availability_strategy = availability_strategy or DefaultFileBasedAvailabilityStrategy(stream_reader)
+        self.availability_strategy = availability_strategy or DefaultFileBasedAvailabilityStrategy(
+            stream_reader
+        )
         self.discovery_policy = discovery_policy
         self.parsers = parsers
         self.validation_policies = validation_policies
-        self.stream_schemas = {s.stream.name: s.stream.json_schema for s in catalog.streams} if catalog else {}
+        self.stream_schemas = (
+            {s.stream.name: s.stream.json_schema for s in catalog.streams} if catalog else {}
+        )
         self.cursor_cls = cursor_cls
         self.logger = init_logger(f"airbyte.{self.name}")
         self.errors_collector: FileBasedErrorsCollector = FileBasedErrorsCollector()
         self._message_repository: Optional[MessageRepository] = None
         concurrent_source = ConcurrentSource.create(
-            MAX_CONCURRENCY, INITIAL_N_PARTITIONS, self.logger, self._slice_logger, self.message_repository
+            MAX_CONCURRENCY,
+            INITIAL_N_PARTITIONS,
+            self.logger,
+            self._slice_logger,
+            self.message_repository,
         )
         self._state = None
         super().__init__(concurrent_source)
@@ -91,10 +120,14 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
     @property
     def message_repository(self) -> MessageRepository:
         if self._message_repository is None:
-            self._message_repository = InMemoryMessageRepository(Level(AirbyteLogFormatter.level_mapping[self.logger.level]))
+            self._message_repository = InMemoryMessageRepository(
+                Level(AirbyteLogFormatter.level_mapping[self.logger.level])
+            )
         return self._message_repository
 
-    def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
+    def check_connection(
+        self, logger: logging.Logger, config: Mapping[str, Any]
+    ) -> Tuple[bool, Optional[Any]]:
         """
         Check that the source can be accessed using the user-provided configuration.
 
@@ -122,20 +155,49 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             )
 
         errors = []
+        tracebacks = []
         for stream in streams:
             if not isinstance(stream, AbstractFileBasedStream):
                 raise ValueError(f"Stream {stream} is not a file-based stream.")
             try:
+                parsed_config = self._get_parsed_config(config)
+                availability_method = (
+                    stream.availability_strategy.check_availability
+                    if self._use_file_transfer(parsed_config)
+                    else stream.availability_strategy.check_availability_and_parsability
+                )
                 (
                     stream_is_available,
                     reason,
-                ) = stream.availability_strategy.check_availability_and_parsability(stream, logger, self)
+                ) = availability_method(stream, logger, self)
+            except AirbyteTracedException as ate:
+                errors.append(f"Unable to connect to stream {stream.name} - {ate.message}")
+                tracebacks.append(traceback.format_exc())
             except Exception:
-                errors.append(f"Unable to connect to stream {stream.name} - {''.join(traceback.format_exc())}")
+                errors.append(f"Unable to connect to stream {stream.name}")
+                tracebacks.append(traceback.format_exc())
             else:
                 if not stream_is_available and reason:
                     errors.append(reason)
 
+        if len(errors) == 1 and len(tracebacks) == 1:
+            raise AirbyteTracedException(
+                internal_message=tracebacks[0],
+                message=f"{errors[0]}",
+                failure_type=FailureType.config_error,
+            )
+        if len(errors) == 1 and len(tracebacks) == 0:
+            raise AirbyteTracedException(
+                message=f"{errors[0]}",
+                failure_type=FailureType.config_error,
+            )
+        elif len(errors) > 1:
+            raise AirbyteTracedException(
+                internal_message="\n".join(tracebacks),
+                message=f"{len(errors)} streams with errors: {', '.join(error for error in errors)}",
+                failure_type=FailureType.config_error,
+            )
+
         return not bool(errors), (errors or None)
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
@@ -144,10 +206,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         """
 
         if self.catalog:
-            state_manager = ConnectorStateManager(
-                stream_instance_map={s.stream.name: s.stream for s in self.catalog.streams},
-                state=self.state,
-            )
+            state_manager = ConnectorStateManager(state=self.state)
         else:
             # During `check` operations we don't have a catalog so cannot create a state manager.
             # Since the state manager is only required for incremental syncs, this is fine.
@@ -169,12 +228,26 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
 
                 sync_mode = self._get_sync_mode_from_catalog(stream_config.name)
 
-                if sync_mode == SyncMode.full_refresh and hasattr(self, "_concurrency_level") and self._concurrency_level is not None:
+                if (
+                    sync_mode == SyncMode.full_refresh
+                    and hasattr(self, "_concurrency_level")
+                    and self._concurrency_level is not None
+                ):
                     cursor = FileBasedFinalStateCursor(
-                        stream_config=stream_config, stream_namespace=None, message_repository=self.message_repository
+                        stream_config=stream_config,
+                        stream_namespace=None,
+                        message_repository=self.message_repository,
                     )
                     stream = FileBasedStreamFacade.create_from_stream(
-                        self._make_default_stream(stream_config, cursor), self, self.logger, stream_state, cursor
+                        stream=self._make_default_stream(
+                            stream_config=stream_config,
+                            cursor=cursor,
+                            parsed_config=parsed_config,
+                        ),
+                        source=self,
+                        logger=self.logger,
+                        state=stream_state,
+                        cursor=cursor,
                     )
 
                 elif (
@@ -197,11 +270,23 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         CursorField(DefaultFileBasedStream.ab_last_mod_col),
                     )
                     stream = FileBasedStreamFacade.create_from_stream(
-                        self._make_default_stream(stream_config, cursor), self, self.logger, stream_state, cursor
+                        stream=self._make_default_stream(
+                            stream_config=stream_config,
+                            cursor=cursor,
+                            parsed_config=parsed_config,
+                        ),
+                        source=self,
+                        logger=self.logger,
+                        state=stream_state,
+                        cursor=cursor,
                     )
                 else:
                     cursor = self.cursor_cls(stream_config)
-                    stream = self._make_default_stream(stream_config, cursor)
+                    stream = self._make_default_stream(
+                        stream_config=stream_config,
+                        cursor=cursor,
+                        parsed_config=parsed_config,
+                    )
 
                 streams.append(stream)
             return streams
@@ -210,7 +295,10 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR) from exc
 
     def _make_default_stream(
-        self, stream_config: FileBasedStreamConfig, cursor: Optional[AbstractFileBasedCursor]
+        self,
+        stream_config: FileBasedStreamConfig,
+        cursor: Optional[AbstractFileBasedCursor],
+        parsed_config: AbstractFileBasedSpec,
     ) -> AbstractFileBasedStream:
         return DefaultFileBasedStream(
             config=stream_config,
@@ -222,9 +310,13 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             validation_policy=self._validate_and_get_validation_policy(stream_config),
             errors_collector=self.errors_collector,
             cursor=cursor,
+            use_file_transfer=self._use_file_transfer(parsed_config),
+            preserve_directory_structure=self._preserve_directory_structure(parsed_config),
         )
 
-    def _get_stream_from_catalog(self, stream_config: FileBasedStreamConfig) -> Optional[AirbyteStream]:
+    def _get_stream_from_catalog(
+        self, stream_config: FileBasedStreamConfig
+    ) -> Optional[AirbyteStream]:
         if self.catalog:
             for stream in self.catalog.streams or []:
                 if stream.stream.name == stream_config.name:
@@ -244,14 +336,16 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         logger: logging.Logger,
         config: Mapping[str, Any],
         catalog: ConfiguredAirbyteCatalog,
-        state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None,
+        state: Optional[List[AirbyteStateMessage]] = None,
     ) -> Iterator[AirbyteMessage]:
         yield from super().read(logger, config, catalog, state)
         # emit all the errors collected
         yield from self.errors_collector.yield_and_raise_collected()
         # count streams using a certain parser
         parsed_config = self._get_parsed_config(config)
-        for parser, count in Counter(stream.format.filetype for stream in parsed_config.streams).items():
+        for parser, count in Counter(
+            stream.format.filetype for stream in parsed_config.streams
+        ).items():
             yield create_analytics_message(f"file-cdk-{parser}-stream-count", count)
 
     def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification:
@@ -267,14 +361,50 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
     def _get_parsed_config(self, config: Mapping[str, Any]) -> AbstractFileBasedSpec:
         return self.spec_class(**config)
 
-    def _validate_and_get_validation_policy(self, stream_config: FileBasedStreamConfig) -> AbstractSchemaValidationPolicy:
+    def _validate_and_get_validation_policy(
+        self, stream_config: FileBasedStreamConfig
+    ) -> AbstractSchemaValidationPolicy:
         if stream_config.validation_policy not in self.validation_policies:
             # This should never happen because we validate the config against the schema's validation_policy enum
             raise ValidationError(
-                f"`validation_policy` must be one of {list(self.validation_policies.keys())}", model=FileBasedStreamConfig
+                f"`validation_policy` must be one of {list(self.validation_policies.keys())}",
+                model=FileBasedStreamConfig,
             )
         return self.validation_policies[stream_config.validation_policy]
 
     def _validate_input_schema(self, stream_config: FileBasedStreamConfig) -> None:
         if stream_config.schemaless and stream_config.input_schema:
-            raise ValidationError("`input_schema` and `schemaless` options cannot both be set", model=FileBasedStreamConfig)
+            raise ValidationError(
+                "`input_schema` and `schemaless` options cannot both be set",
+                model=FileBasedStreamConfig,
+            )
+
+    @staticmethod
+    def _use_file_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
+        use_file_transfer = (
+            hasattr(parsed_config.delivery_method, "delivery_type")
+            and parsed_config.delivery_method.delivery_type == "use_file_transfer"
+        )
+        return use_file_transfer
+
+    @staticmethod
+    def _preserve_directory_structure(parsed_config: AbstractFileBasedSpec) -> bool:
+        """
+        Determines whether to preserve directory structure during file transfer.
+
+        When enabled, files maintain their subdirectory paths in the destination.
+        When disabled, files are flattened to the root of the destination.
+
+        Args:
+            parsed_config: The parsed configuration containing delivery method settings
+
+        Returns:
+            True if directory structure should be preserved (default), False otherwise
+        """
+        if (
+            FileBasedSource._use_file_transfer(parsed_config)
+            and hasattr(parsed_config.delivery_method, "preserve_directory_structure")
+            and parsed_config.delivery_method.preserve_directory_structure is not None
+        ):
+            return parsed_config.delivery_method.preserve_directory_structure
+        return True

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -7,11 +7,13 @@ from abc import ABC, abstractmethod
 from datetime import datetime
 from enum import Enum
 from io import IOBase
-from typing import Iterable, List, Optional, Set
+from os import makedirs, path
+from typing import Any, Dict, Iterable, List, Optional, Set
+
+from wcmatch.glob import GLOBSTAR, globmatch
 
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from wcmatch.glob import GLOBSTAR, globmatch
 
 
 class FileReadMode(Enum):
@@ -44,7 +46,9 @@ class AbstractFileBasedStreamReader(ABC):
         ...
 
     @abstractmethod
-    def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase:
+    def open_file(
+        self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger
+    ) -> IOBase:
         """
         Return a file handle for reading.
 
@@ -79,11 +83,17 @@ class AbstractFileBasedStreamReader(ABC):
         """
         ...
 
-    def filter_files_by_globs_and_start_date(self, files: List[RemoteFile], globs: List[str]) -> Iterable[RemoteFile]:
+    def filter_files_by_globs_and_start_date(
+        self, files: List[RemoteFile], globs: List[str]
+    ) -> Iterable[RemoteFile]:
         """
         Utility method for filtering files based on globs.
         """
-        start_date = datetime.strptime(self.config.start_date, self.DATE_TIME_FORMAT) if self.config and self.config.start_date else None
+        start_date = (
+            datetime.strptime(self.config.start_date, self.DATE_TIME_FORMAT)
+            if self.config and self.config.start_date
+            else None
+        )
         seen = set()
 
         for file in files:
@@ -92,6 +102,16 @@ class AbstractFileBasedStreamReader(ABC):
                     seen.add(file.uri)
                     yield file
 
+    @abstractmethod
+    def file_size(self, file: RemoteFile) -> int:
+        """Utility method to get size of the remote file.
+
+        This is required for connectors that will support writing to
+        files. If the connector does not support writing files, then the
+        subclass can simply `return 0`.
+        """
+        ...
+
     @staticmethod
     def file_matches_globs(file: RemoteFile, globs: List[str]) -> bool:
         # Use the GLOBSTAR flag to enable recursive ** matching
@@ -105,3 +125,61 @@ class AbstractFileBasedStreamReader(ABC):
         """
         prefixes = {glob.split("*")[0] for glob in globs}
         return set(filter(lambda x: bool(x), prefixes))
+
+    def use_file_transfer(self) -> bool:
+        if self.config:
+            use_file_transfer = (
+                hasattr(self.config.delivery_method, "delivery_type")
+                and self.config.delivery_method.delivery_type == "use_file_transfer"
+            )
+            return use_file_transfer
+        return False
+
+    def preserve_directory_structure(self) -> bool:
+        # fall back to preserve subdirectories if config is not present or incomplete
+        if (
+            self.use_file_transfer()
+            and self.config
+            and hasattr(self.config.delivery_method, "preserve_directory_structure")
+            and self.config.delivery_method.preserve_directory_structure is not None
+        ):
+            return self.config.delivery_method.preserve_directory_structure
+        return True
+
+    @abstractmethod
+    def get_file(
+        self, file: RemoteFile, local_directory: str, logger: logging.Logger
+    ) -> Dict[str, Any]:
+        """
+        This is required for connectors that will support writing to
+        files. It will handle the logic to download,get,read,acquire or
+        whatever is more efficient to get a file from the source.
+
+        Args:
+               file (RemoteFile): The remote file object containing URI and metadata.
+               local_directory (str): The local directory path where the file will be downloaded.
+               logger (logging.Logger): Logger for logging information and errors.
+
+           Returns:
+               dict: A dictionary containing the following:
+                   - "file_url" (str): The absolute path of the downloaded file.
+                   - "bytes" (int): The file size in bytes.
+                   - "file_relative_path" (str): The relative path of the file for local storage. Is relative to local_directory as
+                   this a mounted volume in the pod container.
+
+        """
+        ...
+
+    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
+        preserve_directory_structure = self.preserve_directory_structure()
+        if preserve_directory_structure:
+            # Remove left slashes from source path format to make relative path for writing locally
+            file_relative_path = file.uri.lstrip("/")
+        else:
+            file_relative_path = path.basename(file.uri)
+        local_file_path = path.join(local_directory, file_relative_path)
+
+        # Ensure the local directory exists
+        makedirs(path.dirname(local_file_path), exist_ok=True)
+        absolute_file_path = path.abspath(local_file_path)
+        return [file_relative_path, local_file_path, absolute_file_path]

airbyte_cdk/sources/file_based/file_types/__init__.py

@@ -2,12 +2,15 @@ from typing import Any, Mapping, Type
 
 from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat
 from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat
+from airbyte_cdk.sources.file_based.config.excel_format import ExcelFormat
 from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat
 from airbyte_cdk.sources.file_based.config.parquet_format import ParquetFormat
 from airbyte_cdk.sources.file_based.config.unstructured_format import UnstructuredFormat
 
 from .avro_parser import AvroParser
 from .csv_parser import CsvParser
+from .excel_parser import ExcelParser
+from .file_transfer import FileTransfer
 from .file_type_parser import FileTypeParser
 from .jsonl_parser import JsonlParser
 from .parquet_parser import ParquetParser
@@ -16,9 +19,19 @@ from .unstructured_parser import UnstructuredParser
 default_parsers: Mapping[Type[Any], FileTypeParser] = {
     AvroFormat: AvroParser(),
     CsvFormat: CsvParser(),
+    ExcelFormat: ExcelParser(),
     JsonlFormat: JsonlParser(),
     ParquetFormat: ParquetParser(),
     UnstructuredFormat: UnstructuredParser(),
 }
 
-__all__ = ["AvroParser", "CsvParser", "JsonlParser", "ParquetParser", "UnstructuredParser", "default_parsers"]
+__all__ = [
+    "AvroParser",
+    "CsvParser",
+    "ExcelParser",
+    "JsonlParser",
+    "ParquetParser",
+    "UnstructuredParser",
+    "FileTransfer",
+    "default_parsers",
+]