airbyte-cdk 6.45.0__py3-none-any.whl → 6.45.0.dev4101__py3-none-any.whl

airbyte_cdk/sources/declarative/retrievers/file_uploader.py

@@ -0,0 +1,89 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import json
+import logging
+import uuid
+from dataclasses import InitVar, dataclass, field
+from pathlib import Path
+from typing import Any, Mapping, Optional, Union
+
+from airbyte_cdk.models import AirbyteRecordMessageFileReference
+from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import (
+    InterpolatedString,
+)
+from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import (
+    SafeResponse,
+)
+from airbyte_cdk.sources.declarative.requesters import Requester
+from airbyte_cdk.sources.declarative.types import Record, StreamSlice
+from airbyte_cdk.sources.types import Config
+from airbyte_cdk.sources.utils.files_directory import get_files_directory
+
+logger = logging.getLogger("airbyte")
+
+
+@dataclass
+class FileUploader:
+    requester: Requester
+    download_target_extractor: RecordExtractor
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+
+    filename_extractor: Optional[Union[InterpolatedString, str]] = None
+    content_extractor: Optional[RecordExtractor] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if self.filename_extractor:
+            self.filename_extractor = InterpolatedString.create(
+                self.filename_extractor,
+                parameters=parameters,
+            )
+
+    def upload(self, record: Record) -> None:
+        mocked_response = SafeResponse()
+        mocked_response.content = json.dumps(record.data).encode()
+        download_target = list(self.download_target_extractor.extract_records(mocked_response))[0]
+        if not isinstance(download_target, str):
+            raise ValueError(
+                f"download_target is expected to be a str but was {type(download_target)}: {download_target}"
+            )
+
+        response = self.requester.send_request(
+            stream_slice=StreamSlice(
+                partition={}, cursor_slice={}, extra_fields={"download_target": download_target}
+            ),
+        )
+
+        if self.content_extractor:
+            raise NotImplementedError("TODO")
+        else:
+            files_directory = Path(get_files_directory())
+
+            file_name = (
+                self.filename_extractor.eval(self.config, record=record)
+                if self.filename_extractor
+                else str(uuid.uuid4())
+            )
+            file_name = file_name.lstrip("/")
+            file_relative_path = Path(record.stream_name) / Path(file_name)
+
+            full_path = files_directory / file_relative_path
+            full_path.parent.mkdir(parents=True, exist_ok=True)
+
+            with open(str(full_path), "wb") as f:
+                f.write(response.content)
+            file_size_bytes = full_path.stat().st_size
+
+            logger.info("File uploaded successfully")
+            logger.info(f"File url: {str(full_path)}")
+            logger.info(f"File size: {file_size_bytes / 1024} KB")
+            logger.info(f"File relative path: {str(file_relative_path)}")
+
+            record.file_reference = AirbyteRecordMessageFileReference(
+                file_url=str(full_path),
+                file_relative_path=str(file_relative_path),
+                file_size_bytes=file_size_bytes,
+            )

airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -1,9 +1,8 @@
 #
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
 import json
-from collections import defaultdict
 from dataclasses import InitVar, dataclass, field
 from functools import partial
 from itertools import islice
@@ -13,7 +12,6 @@ from typing import (
     Iterable,
     List,
     Mapping,
-    MutableMapping,
     Optional,
     Set,
     Tuple,
@@ -33,7 +31,6 @@ from airbyte_cdk.sources.declarative.partition_routers.single_partition_router i
 )
 from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination
 from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
-from airbyte_cdk.sources.declarative.requesters.query_properties import QueryProperties
 from airbyte_cdk.sources.declarative.requesters.request_options import (
     DefaultRequestOptionsProvider,
     RequestOptionsProvider,
@@ -91,7 +88,6 @@ class SimpleRetriever(Retriever):
     )
     cursor: Optional[DeclarativeCursor] = None
     ignore_stream_slicer_parameters_on_paginated_requests: bool = False
-    additional_query_properties: Optional[QueryProperties] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._paginator = self.paginator or NoPagination(parameters=parameters)
@@ -449,110 +445,43 @@ class SimpleRetriever(Retriever):
         :param stream_slice: The stream slice to read data for
         :return: The records read from the API source
         """
+        _slice = stream_slice or StreamSlice(partition={}, cursor_slice={})  # None-check
 
-        property_chunks = (
-            list(
-                self.additional_query_properties.get_request_property_chunks(
-                    stream_slice=stream_slice
-                )
-            )
-            if self.additional_query_properties
-            else []
+        most_recent_record_from_slice = None
+        record_generator = partial(
+            self._parse_records,
+            stream_slice=stream_slice,
+            stream_state=self.state or {},
+            records_schema=records_schema,
         )
-        records_without_merge_key = []
-        merged_records: MutableMapping[str, Any] = defaultdict(dict)
 
-        _slice = stream_slice or StreamSlice(partition={}, cursor_slice={})  # None-check
-        most_recent_record_from_slice = None
+        if self.cursor and isinstance(self.cursor, ResumableFullRefreshCursor):
+            stream_state = self.state
 
-        if self.additional_query_properties:
-            for properties in property_chunks:
-                _slice = StreamSlice(
-                    partition=_slice.partition or {},
-                    cursor_slice=_slice.cursor_slice or {},
-                    extra_fields={"query_properties": properties},
-                )  # None-check
-
-                record_generator = partial(
-                    self._parse_records,
-                    stream_slice=_slice,
-                    stream_state=self.state or {},
-                    records_schema=records_schema,
+            # Before syncing the RFR stream, we check if the job's prior attempt was successful and don't need to
+            # fetch more records. The platform deletes stream state for full refresh streams before starting a
+            # new job, so we don't need to worry about this value existing for the initial attempt
+            if stream_state.get(FULL_REFRESH_SYNC_COMPLETE_KEY):
+                return
+
+            yield from self._read_single_page(record_generator, stream_state, _slice)
+        else:
+            for stream_data in self._read_pages(record_generator, self.state, _slice):
+                current_record = self._extract_record(stream_data, _slice)
+                if self.cursor and current_record:
+                    self.cursor.observe(_slice, current_record)
+
+                # Latest record read, not necessarily within slice boundaries.
+                # TODO Remove once all custom components implement `observe` method.
+                # https://github.com/airbytehq/airbyte-internal-issues/issues/6955
+                most_recent_record_from_slice = self._get_most_recent_record(
+                    most_recent_record_from_slice, current_record, _slice
                 )
+                yield stream_data
 
-                for stream_data in self._read_pages(record_generator, self.state, _slice):
-                    current_record = self._extract_record(stream_data, _slice)
-                    if self.cursor and current_record:
-                        self.cursor.observe(_slice, current_record)
-
-                    # Latest record read, not necessarily within slice boundaries.
-                    # TODO Remove once all custom components implement `observe` method.
-                    # https://github.com/airbytehq/airbyte-internal-issues/issues/6955
-                    most_recent_record_from_slice = self._get_most_recent_record(
-                        most_recent_record_from_slice, current_record, _slice
-                    )
-
-                    if current_record and self.additional_query_properties.property_chunking:
-                        merge_key = (
-                            self.additional_query_properties.property_chunking.get_merge_key(
-                                current_record
-                            )
-                        )
-                        if merge_key:
-                            merged_records[merge_key].update(current_record)
-                        else:
-                            # We should still emit records even if the record did not have a merge key
-                            records_without_merge_key.append(current_record)
-                    else:
-                        yield stream_data
             if self.cursor:
                 self.cursor.close_slice(_slice, most_recent_record_from_slice)
-
-            if len(merged_records) > 0:
-                yield from [
-                    Record(data=merged_record, stream_name=self.name, associated_slice=stream_slice)
-                    for merged_record in merged_records.values()
-                ]
-            if len(records_without_merge_key) > 0:
-                yield from records_without_merge_key
-        else:
-            _slice = stream_slice or StreamSlice(partition={}, cursor_slice={})  # None-check
-
-            most_recent_record_from_slice = None
-            record_generator = partial(
-                self._parse_records,
-                stream_slice=stream_slice,
-                stream_state=self.state or {},
-                records_schema=records_schema,
-            )
-
-            if self.cursor and isinstance(self.cursor, ResumableFullRefreshCursor):
-                stream_state = self.state
-
-                # Before syncing the RFR stream, we check if the job's prior attempt was successful and don't need to
-                # fetch more records. The platform deletes stream state for full refresh streams before starting a
-                # new job, so we don't need to worry about this value existing for the initial attempt
-                if stream_state.get(FULL_REFRESH_SYNC_COMPLETE_KEY):
-                    return
-
-                yield from self._read_single_page(record_generator, stream_state, _slice)
-            else:
-                for stream_data in self._read_pages(record_generator, self.state, _slice):
-                    current_record = self._extract_record(stream_data, _slice)
-                    if self.cursor and current_record:
-                        self.cursor.observe(_slice, current_record)
-
-                    # Latest record read, not necessarily within slice boundaries.
-                    # TODO Remove once all custom components implement `observe` method.
-                    # https://github.com/airbytehq/airbyte-internal-issues/issues/6955
-                    most_recent_record_from_slice = self._get_most_recent_record(
-                        most_recent_record_from_slice, current_record, _slice
-                    )
-                    yield stream_data
-
-                if self.cursor:
-                    self.cursor.close_slice(_slice, most_recent_record_from_slice)
-            return
+        return
 
     def _get_most_recent_record(
         self,

airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -58,11 +58,16 @@ class DeclarativePartition(Partition):
     def read(self) -> Iterable[Record]:
         for stream_data in self._retriever.read_records(self._json_schema, self._stream_slice):
             if isinstance(stream_data, Mapping):
-                yield Record(
-                    data=stream_data,
-                    stream_name=self.stream_name(),
-                    associated_slice=self._stream_slice,
+                record = (
+                    stream_data
+                    if isinstance(stream_data, Record)
+                    else Record(
+                        data=stream_data,
+                        stream_name=self.stream_name(),
+                        associated_slice=self._stream_slice,
+                    )
                 )
+                yield record
             else:
                 self._message_repository.emit_message(stream_data)
 

airbyte_cdk/sources/declarative/transformations/add_fields.py

@@ -139,9 +139,7 @@ class AddFields(RecordTransformation):
             valid_types = (parsed_field.value_type,) if parsed_field.value_type else None
             value = parsed_field.value.eval(config, valid_types=valid_types, **kwargs)
             is_empty_condition = not self.condition
-            if is_empty_condition or self._filter_interpolator.eval(
-                config, value=value, path=parsed_field.path, **kwargs
-            ):
+            if is_empty_condition or self._filter_interpolator.eval(config, value=value, **kwargs):
                 dpath.new(record, parsed_field.path, value)
 
     def __eq__(self, other: Any) -> bool:

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -8,16 +8,18 @@ from datetime import datetime
 from enum import Enum
 from io import IOBase
 from os import makedirs, path
-from typing import Any, Dict, Iterable, List, Optional, Set
+from typing import Any, Iterable, List, Optional, Set, Tuple, MutableMapping
 
 from wcmatch.glob import GLOBSTAR, globmatch
 
+from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
 from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import (
     include_identities_stream,
     preserve_directory_structure,
     use_file_transfer,
 )
+from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 
 
@@ -28,6 +30,12 @@ class FileReadMode(Enum):
 
 class AbstractFileBasedStreamReader(ABC):
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+    FILE_RELATIVE_PATH = "file_relative_path"
+    FILE_NAME = "file_name"
+    LOCAL_FILE_PATH = "local_file_path"
+    ABSOLUTE_FILE_PATH = "absolute_file_path"
+    SOURCE_FILE_URI = "source_file_relative_path"
+    FILE_FOLDER = "file_folder"
 
     def __init__(self) -> None:
         self._config = None
@@ -148,9 +156,9 @@ class AbstractFileBasedStreamReader(ABC):
         return False
 
     @abstractmethod
-    def get_file(
+    def upload(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
-    ) -> Dict[str, Any]:
+    ) -> Tuple[FileRecordData, AirbyteRecordMessageFileReference]:
         """
         This is required for connectors that will support writing to
         files. It will handle the logic to download,get,read,acquire or
@@ -162,25 +170,35 @@ class AbstractFileBasedStreamReader(ABC):
                logger (logging.Logger): Logger for logging information and errors.
 
            Returns:
-               dict: A dictionary containing the following:
-                   - "file_url" (str): The absolute path of the downloaded file.
-                   - "bytes" (int): The file size in bytes.
-                   - "file_relative_path" (str): The relative path of the file for local storage. Is relative to local_directory as
-                   this a mounted volume in the pod container.
-
+               AirbyteRecordMessageFileReference: A file reference object containing:
+                   - staging_file_url (str): The absolute path to the referenced file in the staging area.
+                   - file_size_bytes (int): The size of the referenced file in bytes.
+                   - source_file_relative_path (str): The relative path to the referenced file in source.
         """
         ...
 
-    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
+    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> MutableMapping[str, Any]:
         preserve_directory_structure = self.preserve_directory_structure()
+        file_uri = file.uri
+        file_name = path.basename(file_uri)
+        file_folder = path.dirname(file_uri)
         if preserve_directory_structure:
             # Remove left slashes from source path format to make relative path for writing locally
-            file_relative_path = file.uri.lstrip("/")
+            file_relative_path = file_uri.lstrip("/")
         else:
-            file_relative_path = path.basename(file.uri)
+            file_relative_path = file_name
         local_file_path = path.join(local_directory, file_relative_path)
-
         # Ensure the local directory exists
         makedirs(path.dirname(local_file_path), exist_ok=True)
         absolute_file_path = path.abspath(local_file_path)
-        return [file_relative_path, local_file_path, absolute_file_path]
+
+        file_paths = {
+            self.FILE_RELATIVE_PATH: file_relative_path,
+            self.LOCAL_FILE_PATH: local_file_path,
+            self.ABSOLUTE_FILE_PATH: absolute_file_path,
+            self.FILE_NAME: file_name,
+            self.FILE_FOLDER: file_folder,
+            self.SOURCE_FILE_URI: file_uri,
+
+        }
+        return file_paths

airbyte_cdk/sources/file_based/file_record_data.py

@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from datetime import datetime
+from typing import Optional
+
+from pydantic.v1 import BaseModel
+
+
+class FileRecordData(BaseModel):
+    """
+    A record in a file-based stream.
+    """
+
+    folder: str
+    filename: str
+    bytes: int
+
+    id: Optional[str] = None
+    created_at: Optional[int] = None
+    updated_at: Optional[int] = None
+    mime_type: Optional[str] = None
+    description: Optional[str] = None

airbyte_cdk/sources/file_based/file_types/file_transfer.py

@@ -2,34 +2,27 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 #
 import logging
-import os
-from typing import Any, Dict, Iterable
+from typing import Iterable, Tuple
 
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
+from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
+from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-
-AIRBYTE_STAGING_DIRECTORY = os.getenv("AIRBYTE_STAGING_DIRECTORY", "/staging/files")
-DEFAULT_LOCAL_DIRECTORY = "/tmp/airbyte-file-transfer"
+from airbyte_cdk.sources.utils.files_directory import get_files_directory
 
 
 class FileTransfer:
     def __init__(self) -> None:
-        self._local_directory = (
-            AIRBYTE_STAGING_DIRECTORY
-            if os.path.exists(AIRBYTE_STAGING_DIRECTORY)
-            else DEFAULT_LOCAL_DIRECTORY
-        )
+        self._local_directory = get_files_directory()
 
-    def get_file(
+    def upload(
         self,
-        config: FileBasedStreamConfig,
         file: RemoteFile,
         stream_reader: AbstractFileBasedStreamReader,
         logger: logging.Logger,
-    ) -> Iterable[Dict[str, Any]]:
+    ) -> Iterable[Tuple[FileRecordData, AirbyteRecordMessageFileReference]]:
         try:
-            yield stream_reader.get_file(
+            yield stream_reader.upload(
                 file=file, local_directory=self._local_directory, logger=logger
             )
         except Exception as ex:

airbyte_cdk/sources/file_based/schema_helpers.py

@@ -18,9 +18,19 @@ JsonSchemaSupportedType = Union[List[str], Literal["string"], str]
 SchemaType = Mapping[str, Mapping[str, JsonSchemaSupportedType]]
 
 schemaless_schema = {"type": "object", "properties": {"data": {"type": "object"}}}
+
 file_transfer_schema = {
     "type": "object",
-    "properties": {"data": {"type": "object"}, "file": {"type": "object"}},
+    "properties": {
+        "folder": {"type": "string"},
+        "file_name": {"type": "string"},
+        "bytes": {"type": "integer"},
+        "id": {"type": ["null", "string"]},
+        "created_at": {"type": ["null", "integer"]},
+        "updated_at": {"type": ["null", "integer"]},
+        "mime_type": {"type": ["null", "string"]},
+        "description": {"type": ["null", "string"]},
+    },
 }
 
 

airbyte_cdk/sources/file_based/stream/concurrent/adapters.py

@@ -270,7 +270,6 @@ class FileBasedStreamPartition(Partition):
                         yield Record(
                             data=record_message_data,
                             stream_name=self.stream_name(),
-                            is_file_transfer_message=self._use_file_transfer(),
                         )
                 else:
                     self._message_repository.emit_message(record_data)

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -11,7 +11,7 @@ from functools import cache
 from os import path
 from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteStream, FailureType, Level
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.exceptions import (
@@ -97,14 +97,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         self, configured_catalog_json_schema: Dict[str, Any]
     ) -> Dict[str, Any]:
         if self.use_file_transfer:
-            return {
-                "type": "object",
-                "properties": {
-                    "file_path": {"type": "string"},
-                    "file_size": {"type": "string"},
-                    self.ab_file_name_col: {"type": "string"},
-                },
-            }
+            return file_transfer_schema
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
@@ -145,14 +138,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         record[self.ab_file_name_col] = file.uri
         return record
 
-    def transform_record_for_file_transfer(
-        self, record: dict[str, Any], file: RemoteFile
-    ) -> dict[str, Any]:
-        # timstamp() returns a float representing the number of seconds since the unix epoch
-        record[self.modified] = int(file.last_modified.timestamp()) * 1000
-        record[self.source_file_url] = file.uri
-        return record
-
     def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
         """
         Yield all records from all remote files in `list_files_for_this_sync`.
@@ -166,6 +151,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             raise MissingSchemaError(FileBasedSourceError.MISSING_SCHEMA, stream=self.name)
         # The stream only supports a single file type, so we can use the same parser for all files
         parser = self.get_parser()
+        file_transfer = FileTransfer()
         for file in stream_slice["files"]:
             # only serialize the datetime once
             file_datetime_string = file.last_modified.strftime(self.DATE_TIME_FORMAT)
@@ -173,19 +159,13 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
 
             try:
                 if self.use_file_transfer:
-                    self.logger.info(f"{self.name}: {file} file-based syncing")
-                    # todo: complete here the code to not rely on local parser
-                    file_transfer = FileTransfer()
-                    for record in file_transfer.get_file(
-                        self.config, file, self.stream_reader, self.logger
+                    for file_record_data, file_reference in file_transfer.upload(
+                        file=file, stream_reader=self.stream_reader, logger=self.logger
                     ):
-                        line_no += 1
-                        if not self.record_passes_validation_policy(record):
-                            n_skipped += 1
-                            continue
-                        record = self.transform_record_for_file_transfer(record, file)
                         yield stream_data_to_airbyte_message(
-                            self.name, record, is_file_transfer_message=True
+                            self.name,
+                            file_record_data.dict(exclude_none=True),
+                            file_reference=file_reference,
                         )
                 else:
                     for record in parser.parse_records(
@@ -259,6 +239,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
 
     @cache
     def get_json_schema(self) -> JsonSchema:
+        if self.use_file_transfer:
+            return file_transfer_schema
         extra_fields = {
             self.ab_last_mod_col: {"type": "string"},
             self.ab_file_name_col: {"type": "string"},
@@ -282,9 +264,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             return {"type": "object", "properties": {**extra_fields, **schema["properties"]}}
 
     def _get_raw_json_schema(self) -> JsonSchema:
-        if self.use_file_transfer:
-            return file_transfer_schema
-        elif self.config.input_schema:
+        if self.config.input_schema:
             return self.config.get_input_schema()  # type: ignore
         elif self.config.schemaless:
             return schemaless_schema
@@ -341,6 +321,11 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             self.config.globs or [], self.config.legacy_prefix, self.logger
         )
 
+    def as_airbyte_stream(self) -> AirbyteStream:
+        file_stream = super().as_airbyte_stream()
+        file_stream.is_file_based = self.use_file_transfer
+        return file_stream
+
     def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]:
         loop = asyncio.get_event_loop()
         schema = loop.run_until_complete(self._infer_schema(files))

airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py

@@ -61,9 +61,7 @@ class PermissionsFileBasedStream(DefaultFileBasedStream):
                 permissions_record = self.transform_record(
                     permissions_record, file, file_datetime_string
                 )
-                yield stream_data_to_airbyte_message(
-                    self.name, permissions_record, is_file_transfer_message=False
-                )
+                yield stream_data_to_airbyte_message(self.name, permissions_record)
             except Exception as e:
                 self.logger.error(f"Failed to retrieve permissions for file {file.uri}: {str(e)}")
                 yield AirbyteMessage(

airbyte_cdk/sources/streams/concurrent/default_stream.py

@@ -29,6 +29,7 @@ class DefaultStream(AbstractStream):
         logger: Logger,
         cursor: Cursor,
         namespace: Optional[str] = None,
+        supports_file_transfer: bool = False,
     ) -> None:
         self._stream_partition_generator = partition_generator
         self._name = name
@@ -39,6 +40,7 @@ class DefaultStream(AbstractStream):
         self._logger = logger
         self._cursor = cursor
         self._namespace = namespace
+        self._supports_file_transfer = supports_file_transfer
 
     def generate_partitions(self) -> Iterable[Partition]:
         yield from self._stream_partition_generator.generate()
@@ -68,6 +70,7 @@ class DefaultStream(AbstractStream):
             json_schema=dict(self._json_schema),
             supported_sync_modes=[SyncMode.full_refresh],
             is_resumable=False,
+            is_file_based=self._supports_file_transfer,
         )
 
         if self._namespace:

airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py

@@ -71,10 +71,6 @@ class AbstractStreamStateConverter(ABC):
         for stream_slice in state.get("slices", []):
             stream_slice[self.START_KEY] = self._from_state_message(stream_slice[self.START_KEY])
             stream_slice[self.END_KEY] = self._from_state_message(stream_slice[self.END_KEY])
-            if self.MOST_RECENT_RECORD_KEY in stream_slice:
-                stream_slice[self.MOST_RECENT_RECORD_KEY] = self._from_state_message(
-                    stream_slice[self.MOST_RECENT_RECORD_KEY]
-                )
         return state
 
     def serialize(