airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev1__py3-none-any.whl

airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py

@@ -11,13 +11,15 @@ from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import (
+    AbstractConcurrentFileBasedCursor,
+)
 from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor
 from airbyte_cdk.sources.file_based.types import StreamState
 from airbyte_cdk.sources.message.repository import MessageRepository
 from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
-from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
+from airbyte_cdk.sources.types import Record
 
 if TYPE_CHECKING:
     from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition
@@ -27,7 +29,9 @@ _NULL_FILE = ""
 
 class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
     CURSOR_FIELD = "_ab_source_file_last_modified"
-    DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL = DefaultFileBasedCursor.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
+    DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL = (
+        DefaultFileBasedCursor.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
+    )
     DEFAULT_MAX_HISTORY_SIZE = 10_000
     DATE_TIME_FORMAT = DefaultFileBasedCursor.DATE_TIME_FORMAT
     zero_value = datetime.min
@@ -51,7 +55,8 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         self._connector_state_manager = connector_state_manager
         self._cursor_field = cursor_field
         self._time_window_if_history_is_full = timedelta(
-            days=stream_config.days_to_sync_if_history_is_full or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
+            days=stream_config.days_to_sync_if_history_is_full
+            or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
         )
         self._state_lock = RLock()
         self._pending_files_lock = RLock()
@@ -70,7 +75,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
     def close_partition(self, partition: Partition) -> None:
         with self._pending_files_lock:
             if self._pending_files is None:
-                raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.")
+                raise RuntimeError(
+                    "Expected pending partitions to be set but it was not. This is unexpected. Please contact Support."
+                )
 
     def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None:
         with self._pending_files_lock:
@@ -81,7 +88,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
                     continue
                 for file in _slice["files"]:
                     if file.uri in self._pending_files.keys():
-                        raise RuntimeError(f"Already found file {_slice} in pending files. This is unexpected. Please contact Support.")
+                        raise RuntimeError(
+                            f"Already found file {_slice} in pending files. This is unexpected. Please contact Support."
+                        )
                 self._pending_files.update({file.uri: file})
 
     def _compute_prev_sync_cursor(self, value: Optional[StreamState]) -> Tuple[datetime, str]:
@@ -96,7 +105,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         # represents the start time that the file was uploaded, we can usually expect that all previous
         # files have already been uploaded. If that's the case, they'll be in history and we'll skip
         # re-uploading them.
-        earliest_file_cursor_value = self._get_cursor_key_from_file(self._compute_earliest_file_in_history())
+        earliest_file_cursor_value = self._get_cursor_key_from_file(
+            self._compute_earliest_file_in_history()
+        )
         cursor_str = min(prev_cursor_str, earliest_file_cursor_value)
         cursor_dt, cursor_uri = cursor_str.split("_", 1)
         return datetime.strptime(cursor_dt, self.DATE_TIME_FORMAT), cursor_uri
@@ -109,8 +120,13 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
     def _compute_earliest_file_in_history(self) -> Optional[RemoteFile]:
         with self._state_lock:
             if self._file_to_datetime_history:
-                filename, last_modified = min(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
-                return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
+                filename, last_modified = min(
+                    self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0])
+                )
+                return RemoteFile(
+                    uri=filename,
+                    last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT),
+                )
             else:
                 return None
 
@@ -120,7 +136,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         :param file: The file to add
         """
         if self._pending_files is None:
-            raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.")
+            raise RuntimeError(
+                "Expected pending partitions to be set but it was not. This is unexpected. Please contact Support."
+            )
         with self._pending_files_lock:
             with self._state_lock:
                 if file.uri not in self._pending_files:
@@ -135,7 +153,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
                     )
                 else:
                     self._pending_files.pop(file.uri)
-                self._file_to_datetime_history[file.uri] = file.last_modified.strftime(self.DATE_TIME_FORMAT)
+                self._file_to_datetime_history[file.uri] = file.last_modified.strftime(
+                    self.DATE_TIME_FORMAT
+                )
                 if len(self._file_to_datetime_history) > self.DEFAULT_MAX_HISTORY_SIZE:
                     # Get the earliest file based on its last modified date and its uri
                     oldest_file = self._compute_earliest_file_in_history()
@@ -155,7 +175,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
                 self._stream_namespace,
                 new_state,
             )
-            state_message = self._connector_state_manager.create_state_message(self._stream_name, self._stream_namespace)
+            state_message = self._connector_state_manager.create_state_message(
+                self._stream_name, self._stream_namespace
+            )
             self._message_repository.emit_message(state_message)
 
     def _get_new_cursor_value(self) -> str:
@@ -183,12 +205,19 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
     def _compute_latest_file_in_history(self) -> Optional[RemoteFile]:
         with self._state_lock:
             if self._file_to_datetime_history:
-                filename, last_modified = max(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
-                return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
+                filename, last_modified = max(
+                    self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0])
+                )
+                return RemoteFile(
+                    uri=filename,
+                    last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT),
+                )
             else:
                 return None
 
-    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+    def get_files_to_sync(
+        self, all_files: Iterable[RemoteFile], logger: logging.Logger
+    ) -> Iterable[RemoteFile]:
         """
         Given the list of files in the source, return the files that should be synced.
         :param all_files: All files in the source
@@ -210,7 +239,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         with self._state_lock:
             if file.uri in self._file_to_datetime_history:
                 # If the file's uri is in the history, we should sync the file if it has been modified since it was synced
-                updated_at_from_history = datetime.strptime(self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT)
+                updated_at_from_history = datetime.strptime(
+                    self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT
+                )
                 if file.last_modified < updated_at_from_history:
                     self._message_repository.emit_message(
                         AirbyteMessage(
@@ -246,7 +277,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         """
         with self._state_lock:
             if self._file_to_datetime_history is None:
-                raise RuntimeError("The history object has not been set. This is unexpected. Please contact Support.")
+                raise RuntimeError(
+                    "The history object has not been set. This is unexpected. Please contact Support."
+                )
             return len(self._file_to_datetime_history) >= self.DEFAULT_MAX_HISTORY_SIZE
 
     def _compute_start_time(self) -> datetime:
@@ -268,7 +301,10 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         Get the state of the cursor.
         """
         with self._state_lock:
-            return {"history": self._file_to_datetime_history, self._cursor_field.cursor_field_key: self._get_new_cursor_value()}
+            return {
+                "history": self._file_to_datetime_history,
+                self._cursor_field.cursor_field_key: self._get_new_cursor_value(),
+            }
 
     def set_initial_state(self, value: StreamState) -> None:
         pass

airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py

@@ -9,12 +9,14 @@ from typing import TYPE_CHECKING, Any, Iterable, List, MutableMapping, Optional
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import (
+    AbstractConcurrentFileBasedCursor,
+)
 from airbyte_cdk.sources.file_based.types import StreamState
 from airbyte_cdk.sources.message import MessageRepository
-from airbyte_cdk.sources.streams import FULL_REFRESH_SENTINEL_STATE_KEY
+from airbyte_cdk.sources.streams import NO_CURSOR_STATE_KEY
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
-from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
+from airbyte_cdk.sources.types import Record
 
 if TYPE_CHECKING:
     from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition
@@ -24,7 +26,11 @@ class FileBasedFinalStateCursor(AbstractConcurrentFileBasedCursor):
     """Cursor that is used to guarantee at least one state message is emitted for a concurrent file-based stream."""
 
     def __init__(
-        self, stream_config: FileBasedStreamConfig, message_repository: MessageRepository, stream_namespace: Optional[str], **kwargs: Any
+        self,
+        stream_config: FileBasedStreamConfig,
+        message_repository: MessageRepository,
+        stream_namespace: Optional[str],
+        **kwargs: Any,
     ):
         self._stream_name = stream_config.name
         self._stream_namespace = stream_namespace
@@ -32,11 +38,11 @@ class FileBasedFinalStateCursor(AbstractConcurrentFileBasedCursor):
         # Normally the connector state manager operates at the source-level. However, we only need it to write the sentinel
         # state message rather than manage overall source state. This is also only temporary as we move to the resumable
         # full refresh world where every stream uses a FileBasedConcurrentCursor with incremental state.
-        self._connector_state_manager = ConnectorStateManager(stream_instance_map={})
+        self._connector_state_manager = ConnectorStateManager()
 
     @property
     def state(self) -> MutableMapping[str, Any]:
-        return {FULL_REFRESH_SENTINEL_STATE_KEY: True}
+        return {NO_CURSOR_STATE_KEY: True}
 
     def observe(self, record: Record) -> None:
         pass
@@ -50,7 +56,9 @@ class FileBasedFinalStateCursor(AbstractConcurrentFileBasedCursor):
     def add_file(self, file: RemoteFile) -> None:
         pass
 
-    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+    def get_files_to_sync(
+        self, all_files: Iterable[RemoteFile], logger: logging.Logger
+    ) -> Iterable[RemoteFile]:
         return all_files
 
     def get_state(self) -> MutableMapping[str, Any]:
@@ -66,6 +74,10 @@ class FileBasedFinalStateCursor(AbstractConcurrentFileBasedCursor):
         pass
 
     def ensure_at_least_one_state_emitted(self) -> None:
-        self._connector_state_manager.update_state_for_stream(self._stream_name, self._stream_namespace, self.state)
-        state_message = self._connector_state_manager.create_state_message(self._stream_name, self._stream_namespace)
+        self._connector_state_manager.update_state_for_stream(
+            self._stream_name, self._stream_namespace, self.state
+        )
+        state_message = self._connector_state_manager.create_state_message(
+            self._stream_name, self._stream_namespace
+        )
         self._message_repository.emit_message(state_message)

airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py

@@ -54,7 +54,9 @@ class AbstractFileBasedCursor(ABC):
         ...
 
     @abstractmethod
-    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+    def get_files_to_sync(
+        self, all_files: Iterable[RemoteFile], logger: logging.Logger
+    ) -> Iterable[RemoteFile]:
         """
         Given the list of files in the source, return the files that should be synced.
         :param all_files: All files in the source

airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py

@@ -8,7 +8,9 @@ from typing import Any, Iterable, MutableMapping, Optional
 
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.file_based.stream.cursor.abstract_file_based_cursor import AbstractFileBasedCursor
+from airbyte_cdk.sources.file_based.stream.cursor.abstract_file_based_cursor import (
+    AbstractFileBasedCursor,
+)
 from airbyte_cdk.sources.file_based.types import StreamState
 
 
@@ -19,14 +21,17 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
     CURSOR_FIELD = "_ab_source_file_last_modified"
 
     def __init__(self, stream_config: FileBasedStreamConfig, **_: Any):
-        super().__init__(stream_config)
+        super().__init__(stream_config)  # type: ignore [safe-super]
         self._file_to_datetime_history: MutableMapping[str, str] = {}
         self._time_window_if_history_is_full = timedelta(
-            days=stream_config.days_to_sync_if_history_is_full or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
+            days=stream_config.days_to_sync_if_history_is_full
+            or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
         )
 
         if self._time_window_if_history_is_full <= timedelta():
-            raise ValueError(f"days_to_sync_if_history_is_full must be a positive timedelta, got {self._time_window_if_history_is_full}")
+            raise ValueError(
+                f"days_to_sync_if_history_is_full must be a positive timedelta, got {self._time_window_if_history_is_full}"
+            )
 
         self._start_time = self._compute_start_time()
         self._initial_earliest_file_in_history: Optional[RemoteFile] = None
@@ -37,7 +42,9 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
         self._initial_earliest_file_in_history = self._compute_earliest_file_in_history()
 
     def add_file(self, file: RemoteFile) -> None:
-        self._file_to_datetime_history[file.uri] = file.last_modified.strftime(self.DATE_TIME_FORMAT)
+        self._file_to_datetime_history[file.uri] = file.last_modified.strftime(
+            self.DATE_TIME_FORMAT
+        )
         if len(self._file_to_datetime_history) > self.DEFAULT_MAX_HISTORY_SIZE:
             # Get the earliest file based on its last modified date and its uri
             oldest_file = self._compute_earliest_file_in_history()
@@ -60,7 +67,9 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
         a string joining the last-modified timestamp of the last synced file and the name of the file.
         """
         if self._file_to_datetime_history.items():
-            filename, timestamp = max(self._file_to_datetime_history.items(), key=lambda x: (x[1], x[0]))
+            filename, timestamp = max(
+                self._file_to_datetime_history.items(), key=lambda x: (x[1], x[0])
+            )
             return f"{timestamp}_{filename}"
         return None
 
@@ -73,7 +82,9 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
     def _should_sync_file(self, file: RemoteFile, logger: logging.Logger) -> bool:
         if file.uri in self._file_to_datetime_history:
             # If the file's uri is in the history, we should sync the file if it has been modified since it was synced
-            updated_at_from_history = datetime.strptime(self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT)
+            updated_at_from_history = datetime.strptime(
+                self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT
+            )
             if file.last_modified < updated_at_from_history:
                 logger.warning(
                     f"The file {file.uri}'s last modified date is older than the last time it was synced. This is unexpected. Skipping the file."
@@ -99,7 +110,9 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
             # The file is not in the history and the history is complete. We know we need to sync the file
             return True
 
-    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+    def get_files_to_sync(
+        self, all_files: Iterable[RemoteFile], logger: logging.Logger
+    ) -> Iterable[RemoteFile]:
         if self._is_history_full():
             logger.warning(
                 f"The state history is full. "
@@ -115,8 +128,12 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
 
     def _compute_earliest_file_in_history(self) -> Optional[RemoteFile]:
         if self._file_to_datetime_history:
-            filename, last_modified = min(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
-            return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
+            filename, last_modified = min(
+                self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0])
+            )
+            return RemoteFile(
+                uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT)
+            )
         else:
             return None
 

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -7,7 +7,7 @@ import itertools
 import traceback
 from copy import deepcopy
 from functools import cache
-from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Set, Union
+from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Union
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level
 from airbyte_cdk.models import Type as MessageType
@@ -16,13 +16,18 @@ from airbyte_cdk.sources.file_based.exceptions import (
     FileBasedSourceError,
     InvalidSchemaError,
     MissingSchemaError,
-    NoFilesMatchingError,
     RecordParseError,
     SchemaInferenceError,
     StopSyncPerValidationPolicy,
 )
+from airbyte_cdk.sources.file_based.file_types import FileTransfer
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.file_based.schema_helpers import SchemaType, merge_schemas, schemaless_schema
+from airbyte_cdk.sources.file_based.schema_helpers import (
+    SchemaType,
+    file_transfer_schema,
+    merge_schemas,
+    schemaless_schema,
+)
 from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream
 from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
 from airbyte_cdk.sources.file_based.types import StreamSlice
@@ -33,17 +38,22 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 
 class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
-
     """
     The default file-based stream.
     """
 
+    FILE_TRANSFER_KW = "use_file_transfer"
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
     ab_last_mod_col = "_ab_source_file_last_modified"
     ab_file_name_col = "_ab_source_file_url"
+    modified = "modified"
+    source_file_url = "source_file_url"
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
+    use_file_transfer = False
 
     def __init__(self, **kwargs: Any):
+        if self.FILE_TRANSFER_KW in kwargs:
+            self.use_file_transfer = kwargs.pop(self.FILE_TRANSFER_KW, False)
         super().__init__(**kwargs)
 
     @property
@@ -55,28 +65,66 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         """State setter, accept state serialized by state getter."""
         self._cursor.set_initial_state(value)
 
-    @property
+    @property  # type: ignore # mypy complains wrong type, but AbstractFileBasedCursor is parent of file-based cursors
     def cursor(self) -> Optional[AbstractFileBasedCursor]:
         return self._cursor
 
     @cursor.setter
     def cursor(self, value: AbstractFileBasedCursor) -> None:
         if self._cursor is not None:
-            raise RuntimeError(f"Cursor for stream {self.name} is already set. This is unexpected. Please contact Support.")
+            raise RuntimeError(
+                f"Cursor for stream {self.name} is already set. This is unexpected. Please contact Support."
+            )
         self._cursor = value
 
     @property
     def primary_key(self) -> PrimaryKeyType:
-        return self.config.primary_key or self.get_parser().get_parser_defined_primary_key(self.config)
+        return self.config.primary_key or self.get_parser().get_parser_defined_primary_key(
+            self.config
+        )
+
+    def _filter_schema_invalid_properties(
+        self, configured_catalog_json_schema: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        if self.use_file_transfer:
+            return {
+                "type": "object",
+                "properties": {
+                    "file_path": {"type": "string"},
+                    "file_size": {"type": "string"},
+                    self.ab_file_name_col: {"type": "string"},
+                },
+            }
+        else:
+            return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
     def compute_slices(self) -> Iterable[Optional[Mapping[str, Any]]]:
         # Sort files by last_modified, uri and return them grouped by last_modified
         all_files = self.list_files()
         files_to_read = self._cursor.get_files_to_sync(all_files, self.logger)
         sorted_files_to_read = sorted(files_to_read, key=lambda f: (f.last_modified, f.uri))
-        slices = [{"files": list(group[1])} for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)]
+        slices = [
+            {"files": list(group[1])}
+            for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)
+        ]
         return slices
 
+    def transform_record(
+        self, record: dict[str, Any], file: RemoteFile, last_updated: str
+    ) -> dict[str, Any]:
+        # adds _ab_source_file_last_modified and _ab_source_file_url to the record
+        record[self.ab_last_mod_col] = last_updated
+        record[self.ab_file_name_col] = file.uri
+        return record
+
+    def transform_record_for_file_transfer(
+        self, record: dict[str, Any], file: RemoteFile
+    ) -> dict[str, Any]:
+        # timstamp() returns a float representing the number of seconds since the unix epoch
+        record[self.modified] = int(file.last_modified.timestamp()) * 1000
+        record[self.source_file_url] = file.uri
+        return record
+
     def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
         """
         Yield all records from all remote files in `list_files_for_this_sync`.
@@ -96,16 +144,33 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             n_skipped = line_no = 0
 
             try:
-                for record in parser.parse_records(self.config, file, self.stream_reader, self.logger, schema):
-                    line_no += 1
-                    if self.config.schemaless:
-                        record = {"data": record}
-                    elif not self.record_passes_validation_policy(record):
-                        n_skipped += 1
-                        continue
-                    record[self.ab_last_mod_col] = file_datetime_string
-                    record[self.ab_file_name_col] = file.uri
-                    yield stream_data_to_airbyte_message(self.name, record)
+                if self.use_file_transfer:
+                    self.logger.info(f"{self.name}: {file} file-based syncing")
+                    # todo: complete here the code to not rely on local parser
+                    file_transfer = FileTransfer()
+                    for record in file_transfer.get_file(
+                        self.config, file, self.stream_reader, self.logger
+                    ):
+                        line_no += 1
+                        if not self.record_passes_validation_policy(record):
+                            n_skipped += 1
+                            continue
+                        record = self.transform_record_for_file_transfer(record, file)
+                        yield stream_data_to_airbyte_message(
+                            self.name, record, is_file_transfer_message=True
+                        )
+                else:
+                    for record in parser.parse_records(
+                        self.config, file, self.stream_reader, self.logger, schema
+                    ):
+                        line_no += 1
+                        if self.config.schemaless:
+                            record = {"data": record}
+                        elif not self.record_passes_validation_policy(record):
+                            n_skipped += 1
+                            continue
+                        record = self.transform_record(record, file, file_datetime_string)
+                        yield stream_data_to_airbyte_message(self.name, record)
                 self._cursor.add_file(file)
 
             except StopSyncPerValidationPolicy:
@@ -172,49 +237,71 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         }
         try:
             schema = self._get_raw_json_schema()
-        except (InvalidSchemaError, NoFilesMatchingError) as config_exception:
-            self.logger.exception(FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value, exc_info=config_exception)
+        except InvalidSchemaError as config_exception:
             raise AirbyteTracedException(
                 internal_message="Please check the logged errors for more information.",
                 message=FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value,
                 exception=AirbyteTracedException(exception=config_exception),
                 failure_type=FailureType.config_error,
             )
+        except AirbyteTracedException as ate:
+            raise ate
         except Exception as exc:
-            raise SchemaInferenceError(FileBasedSourceError.SCHEMA_INFERENCE_ERROR, stream=self.name) from exc
+            raise SchemaInferenceError(
+                FileBasedSourceError.SCHEMA_INFERENCE_ERROR, stream=self.name
+            ) from exc
         else:
             return {"type": "object", "properties": {**extra_fields, **schema["properties"]}}
 
     def _get_raw_json_schema(self) -> JsonSchema:
-        if self.config.input_schema:
+        if self.use_file_transfer:
+            return file_transfer_schema
+        elif self.config.input_schema:
             return self.config.get_input_schema()  # type: ignore
         elif self.config.schemaless:
             return schemaless_schema
         else:
             files = self.list_files()
-            total_n_files = len(files)
+            first_n_files = len(files)
 
-            if total_n_files == 0:
-                raise NoFilesMatchingError(FileBasedSourceError.EMPTY_STREAM, stream=self.name)
-
-            max_n_files_for_schema_inference = self._discovery_policy.get_max_n_files_for_schema_inference(self.get_parser())
-            if total_n_files > max_n_files_for_schema_inference:
-                # Use the most recent files for schema inference, so we pick up schema changes during discovery.
-                files = sorted(files, key=lambda x: x.last_modified, reverse=True)[:max_n_files_for_schema_inference]
-                self.logger.warn(
-                    msg=f"Refusing to infer schema for all {total_n_files} files; using {max_n_files_for_schema_inference} files."
+            if self.config.recent_n_files_to_read_for_schema_discovery:
+                self.logger.info(
+                    msg=(
+                        f"Only first {self.config.recent_n_files_to_read_for_schema_discovery} files will be used to infer schema "
+                        f"for stream {self.name} due to limitation in config."
+                    )
                 )
+                first_n_files = self.config.recent_n_files_to_read_for_schema_discovery
+
+        if first_n_files == 0:
+            self.logger.warning(
+                msg=f"No files were identified in the stream {self.name}. Setting default schema for the stream."
+            )
+            return schemaless_schema
 
-            inferred_schema = self.infer_schema(files)
+        max_n_files_for_schema_inference = (
+            self._discovery_policy.get_max_n_files_for_schema_inference(self.get_parser())
+        )
 
-            if not inferred_schema:
-                raise InvalidSchemaError(
-                    FileBasedSourceError.INVALID_SCHEMA_ERROR,
-                    details=f"Empty schema. Please check that the files are valid for format {self.config.format}",
-                    stream=self.name,
-                )
+        if first_n_files > max_n_files_for_schema_inference:
+            # Use the most recent files for schema inference, so we pick up schema changes during discovery.
+            self.logger.warning(
+                msg=f"Refusing to infer schema for {first_n_files} files; using {max_n_files_for_schema_inference} files."
+            )
+            first_n_files = max_n_files_for_schema_inference
+
+        files = sorted(files, key=lambda x: x.last_modified, reverse=True)[:first_n_files]
+
+        inferred_schema = self.infer_schema(files)
 
-            schema = {"type": "object", "properties": inferred_schema}
+        if not inferred_schema:
+            raise InvalidSchemaError(
+                FileBasedSourceError.INVALID_SCHEMA_ERROR,
+                details=f"Empty schema. Please check that the files are valid for format {self.config.format}",
+                stream=self.name,
+            )
+
+        schema = {"type": "object", "properties": inferred_schema}
 
         return schema
 
@@ -222,7 +309,9 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         """
         Return all files that belong to the stream as defined by the stream's globs.
         """
-        return self.stream_reader.get_matching_files(self.config.globs or [], self.config.legacy_prefix, self.logger)
+        return self.stream_reader.get_matching_files(
+            self.config.globs or [], self.config.legacy_prefix, self.logger
+        )
 
     def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]:
         loop = asyncio.get_event_loop()
@@ -260,23 +349,36 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         n_started, n_files = 0, len(files)
         files_iterator = iter(files)
         while pending_tasks or n_started < n_files:
-            while len(pending_tasks) <= self._discovery_policy.n_concurrent_requests and (file := next(files_iterator, None)):
+            while len(pending_tasks) <= self._discovery_policy.n_concurrent_requests and (
+                file := next(files_iterator, None)
+            ):
                 pending_tasks.add(asyncio.create_task(self._infer_file_schema(file)))
                 n_started += 1
             # Return when the first task is completed so that we can enqueue a new task as soon as the
             # number of concurrent tasks drops below the number allowed.
-            done, pending_tasks = await asyncio.wait(pending_tasks, return_when=asyncio.FIRST_COMPLETED)
+            done, pending_tasks = await asyncio.wait(
+                pending_tasks, return_when=asyncio.FIRST_COMPLETED
+            )
             for task in done:
                 try:
                     base_schema = merge_schemas(base_schema, task.result())
+                except AirbyteTracedException as ate:
+                    raise ate
                 except Exception as exc:
-                    self.logger.error(f"An error occurred inferring the schema. \n {traceback.format_exc()}", exc_info=exc)
+                    self.logger.error(
+                        f"An error occurred inferring the schema. \n {traceback.format_exc()}",
+                        exc_info=exc,
+                    )
 
         return base_schema
 
     async def _infer_file_schema(self, file: RemoteFile) -> SchemaType:
         try:
-            return await self.get_parser().infer_schema(self.config, file, self.stream_reader, self.logger)
+            return await self.get_parser().infer_schema(
+                self.config, file, self.stream_reader, self.logger
+            )
+        except AirbyteTracedException as ate:
+            raise ate
         except Exception as exc:
             raise SchemaInferenceError(
                 FileBasedSourceError.SCHEMA_INFERENCE_ERROR,