airbyte-cdk 6.44.0__py3-none-any.whl → 6.45.0.dev4100__py3-none-any.whl

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -54,11 +54,7 @@ from airbyte_cdk.sources.declarative.auth.token_provider import (
     SessionTokenProvider,
     TokenProvider,
 )
-from airbyte_cdk.sources.declarative.checks import (
-    CheckDynamicStream,
-    CheckStream,
-    DynamicStreamCheckConfig,
-)
+from airbyte_cdk.sources.declarative.checks import CheckDynamicStream, CheckStream
 from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel
 from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
 from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
@@ -106,7 +102,6 @@ from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_mi
 )
 from airbyte_cdk.sources.declarative.models import (
     CustomStateMigration,
-    GzipDecoder,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     AddedFieldDefinition as AddedFieldDefinitionModel,
@@ -223,10 +218,10 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
     DynamicSchemaLoader as DynamicSchemaLoaderModel,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    DynamicStreamCheckConfig as DynamicStreamCheckConfigModel,
+    ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
+    FileUploader as FileUploaderModel,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel,
@@ -234,9 +229,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     FlattenFields as FlattenFieldsModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    GroupingPartitionRouter as GroupingPartitionRouterModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     GzipDecoder as GzipDecoderModel,
 )
@@ -395,7 +387,6 @@ from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import (
 )
 from airbyte_cdk.sources.declarative.partition_routers import (
     CartesianProductStreamSlicer,
-    GroupingPartitionRouter,
     ListPartitionRouter,
     PartitionRouter,
     SinglePartitionRouter,
@@ -453,6 +444,7 @@ from airbyte_cdk.sources.declarative.retrievers import (
     SimpleRetriever,
     SimpleRetrieverTestReadDecorator,
 )
+from airbyte_cdk.sources.declarative.retrievers.file_uploader import FileUploader
 from airbyte_cdk.sources.declarative.schema import (
     ComplexFieldType,
     DefaultSchemaLoader,
@@ -566,7 +558,6 @@ class ModelToComponentFactory:
             BasicHttpAuthenticatorModel: self.create_basic_http_authenticator,
             BearerAuthenticatorModel: self.create_bearer_authenticator,
             CheckStreamModel: self.create_check_stream,
-            DynamicStreamCheckConfigModel: self.create_dynamic_stream_check_config,
             CheckDynamicStreamModel: self.create_check_dynamic_stream,
             CompositeErrorHandlerModel: self.create_composite_error_handler,
             ConcurrencyLevelModel: self.create_concurrency_level,
@@ -645,12 +636,12 @@ class ModelToComponentFactory:
             ComponentMappingDefinitionModel: self.create_components_mapping_definition,
             ZipfileDecoderModel: self.create_zipfile_decoder,
             HTTPAPIBudgetModel: self.create_http_api_budget,
+            FileUploaderModel: self.create_file_uploader,
             FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy,
             MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy,
             UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
             RateModel: self.create_rate,
             HttpRequestRegexMatcherModel: self.create_http_request_matcher,
-            GroupingPartitionRouterModel: self.create_grouping_partition_router,
         }
 
         # Needed for the case where we need to perform a second parse on the fields of a custom component
@@ -944,36 +935,8 @@ class ModelToComponentFactory:
         )
 
     @staticmethod
-    def create_dynamic_stream_check_config(
-        model: DynamicStreamCheckConfigModel, config: Config, **kwargs: Any
-    ) -> DynamicStreamCheckConfig:
-        return DynamicStreamCheckConfig(
-            dynamic_stream_name=model.dynamic_stream_name,
-            stream_count=model.stream_count or 0,
-        )
-
-    def create_check_stream(
-        self, model: CheckStreamModel, config: Config, **kwargs: Any
-    ) -> CheckStream:
-        if model.dynamic_streams_check_configs is None and model.stream_names is None:
-            raise ValueError(
-                "Expected either stream_names or dynamic_streams_check_configs to be set for CheckStream"
-            )
-
-        dynamic_streams_check_configs = (
-            [
-                self._create_component_from_model(model=dynamic_stream_check_config, config=config)
-                for dynamic_stream_check_config in model.dynamic_streams_check_configs
-            ]
-            if model.dynamic_streams_check_configs
-            else []
-        )
-
-        return CheckStream(
-            stream_names=model.stream_names or [],
-            dynamic_streams_check_configs=dynamic_streams_check_configs,
-            parameters={},
-        )
+    def create_check_stream(model: CheckStreamModel, config: Config, **kwargs: Any) -> CheckStream:
+        return CheckStream(stream_names=model.stream_names, parameters={})
 
     @staticmethod
     def create_check_dynamic_stream(
@@ -1396,9 +1359,6 @@ class ModelToComponentFactory:
         )
         stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
 
-        # Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state
-        use_global_cursor = isinstance(partition_router, GroupingPartitionRouter)
-
         # Return the concurrent cursor and state converter
         return ConcurrentPerPartitionCursor(
             cursor_factory=cursor_factory,
@@ -1410,7 +1370,6 @@ class ModelToComponentFactory:
             connector_state_manager=state_manager,
             connector_state_converter=connector_state_converter,
             cursor_field=cursor_field,
-            use_global_cursor=use_global_cursor,
         )
 
     @staticmethod
@@ -1796,6 +1755,11 @@ class ModelToComponentFactory:
                 transformations.append(
                     self._create_component_from_model(model=transformation_model, config=config)
                 )
+        file_uploader = None
+        if model.file_uploader:
+            file_uploader = self._create_component_from_model(
+                model=model.file_uploader, config=config
+            )
 
         retriever = self._create_component_from_model(
             model=model.retriever,
@@ -1807,6 +1771,7 @@ class ModelToComponentFactory:
             stop_condition_on_cursor=stop_condition_on_cursor,
             client_side_incremental_sync=client_side_incremental_sync,
             transformations=transformations,
+            file_uploader=file_uploader,
             incremental_sync=model.incremental_sync,
         )
         cursor_field = model.incremental_sync.cursor_field if model.incremental_sync else None
@@ -2648,6 +2613,7 @@ class ModelToComponentFactory:
         transformations: List[RecordTransformation] | None = None,
         decoder: Decoder | None = None,
         client_side_incremental_sync: Dict[str, Any] | None = None,
+        file_uploader: Optional[FileUploader] = None,
         **kwargs: Any,
     ) -> RecordSelector:
         extractor = self._create_component_from_model(
@@ -2685,6 +2651,7 @@ class ModelToComponentFactory:
             config=config,
             record_filter=record_filter,
             transformations=transformations or [],
+            file_uploader=file_uploader,
             schema_normalization=schema_normalization,
             parameters=model.parameters or {},
             transform_before_filtering=transform_before_filtering,
@@ -2742,6 +2709,7 @@ class ModelToComponentFactory:
         stop_condition_on_cursor: bool = False,
         client_side_incremental_sync: Optional[Dict[str, Any]] = None,
         transformations: List[RecordTransformation],
+        file_uploader: Optional[FileUploader] = None,
         incremental_sync: Optional[
             Union[
                 IncrementingCountCursorModel, DatetimeBasedCursorModel, CustomIncrementalSyncModel
@@ -2764,6 +2732,7 @@ class ModelToComponentFactory:
             decoder=decoder,
             transformations=transformations,
             client_side_incremental_sync=client_side_incremental_sync,
+            file_uploader=file_uploader,
         )
         url_base = (
             model.requester.url_base
@@ -3118,11 +3087,8 @@ class ModelToComponentFactory:
                 stream_slices,
                 self._job_tracker,
                 self._message_repository,
-                # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
                 has_bulk_parent=False,
-                # set the `job_max_retry` to 1 for the `Connector Builder`` use-case.
-                # `None` == default retry is set to 3 attempts, under the hood.
-                job_max_retry=1 if self._emit_connector_builder_messages else None,
+                # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
             ),
             stream_slicer=stream_slicer,
             config=config,
@@ -3366,6 +3332,30 @@ class ModelToComponentFactory:
             matchers=matchers,
         )
 
+    def create_file_uploader(
+        self, model: FileUploaderModel, config: Config, **kwargs: Any
+    ) -> FileUploader:
+        name = "File Uploader"
+        requester = self._create_component_from_model(
+            model=model.requester,
+            config=config,
+            name=name,
+            **kwargs,
+        )
+        download_target_extractor = self._create_component_from_model(
+            model=model.download_target_extractor,
+            config=config,
+            name=name,
+            **kwargs,
+        )
+        return FileUploader(
+            requester=requester,
+            download_target_extractor=download_target_extractor,
+            config=config,
+            parameters=model.parameters or {},
+            filename_extractor=model.filename_extractor if model.filename_extractor else None,
+        )
+
     def create_moving_window_call_rate_policy(
         self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any
     ) -> MovingWindowCallRatePolicy:
@@ -3415,34 +3405,3 @@ class ModelToComponentFactory:
         self._api_budget = self.create_component(
             model_type=HTTPAPIBudgetModel, component_definition=component_definition, config=config
         )
-
-    def create_grouping_partition_router(
-        self, model: GroupingPartitionRouterModel, config: Config, **kwargs: Any
-    ) -> GroupingPartitionRouter:
-        underlying_router = self._create_component_from_model(
-            model=model.underlying_partition_router, config=config
-        )
-        if model.group_size < 1:
-            raise ValueError(f"Group size must be greater than 0, got {model.group_size}")
-
-        # Request options in underlying partition routers are not supported for GroupingPartitionRouter
-        # because they are specific to individual partitions and cannot be aggregated or handled
-        # when grouping, potentially leading to incorrect API calls. Any request customization
-        # should be managed at the stream level through the requester's configuration.
-        if isinstance(underlying_router, SubstreamPartitionRouter):
-            if any(
-                parent_config.request_option
-                for parent_config in underlying_router.parent_stream_configs
-            ):
-                raise ValueError("Request options are not supported for GroupingPartitionRouter.")
-
-        if isinstance(underlying_router, ListPartitionRouter):
-            if underlying_router.request_option:
-                raise ValueError("Request options are not supported for GroupingPartitionRouter.")
-
-        return GroupingPartitionRouter(
-            group_size=model.group_size,
-            underlying_partition_router=underlying_router,
-            deduplicate=model.deduplicate if model.deduplicate is not None else True,
-            config=config,
-        )

airbyte_cdk/sources/declarative/partition_routers/__init__.py

@@ -8,9 +8,6 @@ from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_route
 from airbyte_cdk.sources.declarative.partition_routers.cartesian_product_stream_slicer import (
     CartesianProductStreamSlicer,
 )
-from airbyte_cdk.sources.declarative.partition_routers.grouping_partition_router import (
-    GroupingPartitionRouter,
-)
 from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import (
     ListPartitionRouter,
 )
@@ -25,7 +22,6 @@ from airbyte_cdk.sources.declarative.partition_routers.substream_partition_route
 __all__ = [
     "AsyncJobPartitionRouter",
     "CartesianProductStreamSlicer",
-    "GroupingPartitionRouter",
     "ListPartitionRouter",
     "SinglePartitionRouter",
     "SubstreamPartitionRouter",

airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py

@@ -374,11 +374,7 @@ class SubstreamPartitionRouter(PartitionRouter):
         # Ignore per-partition states or invalid formats.
         if isinstance(substream_state, (list, dict)) or len(substream_state_values) != 1:
             # If a global state is present under the key "state", use its first value.
-            if (
-                "state" in stream_state
-                and isinstance(stream_state["state"], dict)
-                and stream_state["state"] != {}
-            ):
+            if "state" in stream_state and isinstance(stream_state["state"], dict):
                 substream_state = list(stream_state["state"].values())[0]
             else:
                 return {}

airbyte_cdk/sources/declarative/retrievers/file_uploader.py

@@ -0,0 +1,89 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import json
+import logging
+import uuid
+from dataclasses import InitVar, dataclass, field
+from pathlib import Path
+from typing import Any, Mapping, Optional, Union
+
+from airbyte_cdk.models import AirbyteRecordMessageFileReference
+from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import (
+    InterpolatedString,
+)
+from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import (
+    SafeResponse,
+)
+from airbyte_cdk.sources.declarative.requesters import Requester
+from airbyte_cdk.sources.declarative.types import Record, StreamSlice
+from airbyte_cdk.sources.types import Config
+from airbyte_cdk.sources.utils.files_directory import get_files_directory
+
+logger = logging.getLogger("airbyte")
+
+
+@dataclass
+class FileUploader:
+    requester: Requester
+    download_target_extractor: RecordExtractor
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+
+    filename_extractor: Optional[Union[InterpolatedString, str]] = None
+    content_extractor: Optional[RecordExtractor] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if self.filename_extractor:
+            self.filename_extractor = InterpolatedString.create(
+                self.filename_extractor,
+                parameters=parameters,
+            )
+
+    def upload(self, record: Record) -> None:
+        mocked_response = SafeResponse()
+        mocked_response.content = json.dumps(record.data).encode()
+        download_target = list(self.download_target_extractor.extract_records(mocked_response))[0]
+        if not isinstance(download_target, str):
+            raise ValueError(
+                f"download_target is expected to be a str but was {type(download_target)}: {download_target}"
+            )
+
+        response = self.requester.send_request(
+            stream_slice=StreamSlice(
+                partition={}, cursor_slice={}, extra_fields={"download_target": download_target}
+            ),
+        )
+
+        if self.content_extractor:
+            raise NotImplementedError("TODO")
+        else:
+            files_directory = Path(get_files_directory())
+
+            file_name = (
+                self.filename_extractor.eval(self.config, record=record)
+                if self.filename_extractor
+                else str(uuid.uuid4())
+            )
+            file_name = file_name.lstrip("/")
+            file_relative_path = Path(record.stream_name) / Path(file_name)
+
+            full_path = files_directory / file_relative_path
+            full_path.parent.mkdir(parents=True, exist_ok=True)
+
+            with open(str(full_path), "wb") as f:
+                f.write(response.content)
+            file_size_bytes = full_path.stat().st_size
+
+            logger.info("File uploaded successfully")
+            logger.info(f"File url: {str(full_path)}")
+            logger.info(f"File size: {file_size_bytes / 1024} KB")
+            logger.info(f"File relative path: {str(file_relative_path)}")
+
+            record.file_reference = AirbyteRecordMessageFileReference(
+                file_url=str(full_path),
+                file_relative_path=str(file_relative_path),
+                file_size_bytes=file_size_bytes,
+            )

airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -58,11 +58,16 @@ class DeclarativePartition(Partition):
     def read(self) -> Iterable[Record]:
         for stream_data in self._retriever.read_records(self._json_schema, self._stream_slice):
             if isinstance(stream_data, Mapping):
-                yield Record(
-                    data=stream_data,
-                    stream_name=self.stream_name(),
-                    associated_slice=self._stream_slice,
+                record = (
+                    stream_data
+                    if isinstance(stream_data, Record)
+                    else Record(
+                        data=stream_data,
+                        stream_name=self.stream_name(),
+                        associated_slice=self._stream_slice,
+                    )
                 )
+                yield record
             else:
                 self._message_repository.emit_message(stream_data)
 

airbyte_cdk/sources/declarative/transformations/add_fields.py

@@ -139,9 +139,7 @@ class AddFields(RecordTransformation):
             valid_types = (parsed_field.value_type,) if parsed_field.value_type else None
             value = parsed_field.value.eval(config, valid_types=valid_types, **kwargs)
             is_empty_condition = not self.condition
-            if is_empty_condition or self._filter_interpolator.eval(
-                config, value=value, path=parsed_field.path, **kwargs
-            ):
+            if is_empty_condition or self._filter_interpolator.eval(config, value=value, **kwargs):
                 dpath.new(record, parsed_field.path, value)
 
     def __eq__(self, other: Any) -> bool:

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -8,16 +8,18 @@ from datetime import datetime
 from enum import Enum
 from io import IOBase
 from os import makedirs, path
-from typing import Any, Dict, Iterable, List, Optional, Set
+from typing import Iterable, List, Optional, Set, Tuple
 
 from wcmatch.glob import GLOBSTAR, globmatch
 
+from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
 from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import (
     include_identities_stream,
     preserve_directory_structure,
     use_file_transfer,
 )
+from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 
 
@@ -148,9 +150,9 @@ class AbstractFileBasedStreamReader(ABC):
         return False
 
     @abstractmethod
-    def get_file(
+    def upload(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
-    ) -> Dict[str, Any]:
+    ) -> Tuple[FileRecordData, AirbyteRecordMessageFileReference]:
         """
         This is required for connectors that will support writing to
         files. It will handle the logic to download,get,read,acquire or
@@ -162,12 +164,10 @@ class AbstractFileBasedStreamReader(ABC):
                logger (logging.Logger): Logger for logging information and errors.
 
            Returns:
-               dict: A dictionary containing the following:
-                   - "file_url" (str): The absolute path of the downloaded file.
-                   - "bytes" (int): The file size in bytes.
-                   - "file_relative_path" (str): The relative path of the file for local storage. Is relative to local_directory as
-                   this a mounted volume in the pod container.
-
+               AirbyteRecordMessageFileReference: A file reference object containing:
+                   - staging_file_url (str): The absolute path to the referenced file in the staging area.
+                   - file_size_bytes (int): The size of the referenced file in bytes.
+                   - source_file_relative_path (str): The relative path to the referenced file in source.
         """
         ...
 

airbyte_cdk/sources/file_based/file_record_data.py

@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from datetime import datetime
+from typing import Optional
+
+from pydantic.v1 import BaseModel
+
+
+class FileRecordData(BaseModel):
+    """
+    A record in a file-based stream.
+    """
+
+    folder: str
+    filename: str
+    bytes: int
+
+    id: Optional[str] = None
+    created_at: Optional[int] = None
+    updated_at: Optional[int] = None
+    mime_type: Optional[str] = None
+    description: Optional[str] = None

airbyte_cdk/sources/file_based/file_types/file_transfer.py

@@ -2,34 +2,27 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 #
 import logging
-import os
-from typing import Any, Dict, Iterable
+from typing import Iterable, Tuple
 
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
+from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
+from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-
-AIRBYTE_STAGING_DIRECTORY = os.getenv("AIRBYTE_STAGING_DIRECTORY", "/staging/files")
-DEFAULT_LOCAL_DIRECTORY = "/tmp/airbyte-file-transfer"
+from airbyte_cdk.sources.utils.files_directory import get_files_directory
 
 
 class FileTransfer:
     def __init__(self) -> None:
-        self._local_directory = (
-            AIRBYTE_STAGING_DIRECTORY
-            if os.path.exists(AIRBYTE_STAGING_DIRECTORY)
-            else DEFAULT_LOCAL_DIRECTORY
-        )
+        self._local_directory = get_files_directory()
 
-    def get_file(
+    def upload(
         self,
-        config: FileBasedStreamConfig,
         file: RemoteFile,
         stream_reader: AbstractFileBasedStreamReader,
         logger: logging.Logger,
-    ) -> Iterable[Dict[str, Any]]:
+    ) -> Iterable[Tuple[FileRecordData, AirbyteRecordMessageFileReference]]:
         try:
-            yield stream_reader.get_file(
+            yield stream_reader.upload(
                 file=file, local_directory=self._local_directory, logger=logger
             )
         except Exception as ex:

airbyte_cdk/sources/file_based/schema_helpers.py

@@ -18,9 +18,19 @@ JsonSchemaSupportedType = Union[List[str], Literal["string"], str]
 SchemaType = Mapping[str, Mapping[str, JsonSchemaSupportedType]]
 
 schemaless_schema = {"type": "object", "properties": {"data": {"type": "object"}}}
+
 file_transfer_schema = {
     "type": "object",
-    "properties": {"data": {"type": "object"}, "file": {"type": "object"}},
+    "properties": {
+        "folder": {"type": "string"},
+        "file_name": {"type": "string"},
+        "bytes": {"type": "integer"},
+        "id": {"type": ["null", "string"]},
+        "created_at": {"type": ["null", "integer"]},
+        "updated_at": {"type": ["null", "integer"]},
+        "mime_type": {"type": ["null", "string"]},
+        "description": {"type": ["null", "string"]},
+    },
 }
 
 

airbyte_cdk/sources/file_based/stream/concurrent/adapters.py

@@ -270,7 +270,6 @@ class FileBasedStreamPartition(Partition):
                         yield Record(
                             data=record_message_data,
                             stream_name=self.stream_name(),
-                            is_file_transfer_message=self._use_file_transfer(),
                         )
                 else:
                     self._message_repository.emit_message(record_data)

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -11,7 +11,7 @@ from functools import cache
 from os import path
 from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteStream, FailureType, Level
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.exceptions import (
@@ -97,14 +97,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         self, configured_catalog_json_schema: Dict[str, Any]
     ) -> Dict[str, Any]:
         if self.use_file_transfer:
-            return {
-                "type": "object",
-                "properties": {
-                    "file_path": {"type": "string"},
-                    "file_size": {"type": "string"},
-                    self.ab_file_name_col: {"type": "string"},
-                },
-            }
+            return file_transfer_schema
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
@@ -145,14 +138,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         record[self.ab_file_name_col] = file.uri
         return record
 
-    def transform_record_for_file_transfer(
-        self, record: dict[str, Any], file: RemoteFile
-    ) -> dict[str, Any]:
-        # timstamp() returns a float representing the number of seconds since the unix epoch
-        record[self.modified] = int(file.last_modified.timestamp()) * 1000
-        record[self.source_file_url] = file.uri
-        return record
-
     def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
         """
         Yield all records from all remote files in `list_files_for_this_sync`.
@@ -166,6 +151,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             raise MissingSchemaError(FileBasedSourceError.MISSING_SCHEMA, stream=self.name)
         # The stream only supports a single file type, so we can use the same parser for all files
         parser = self.get_parser()
+        file_transfer = FileTransfer()
         for file in stream_slice["files"]:
             # only serialize the datetime once
             file_datetime_string = file.last_modified.strftime(self.DATE_TIME_FORMAT)
@@ -173,19 +159,13 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
 
             try:
                 if self.use_file_transfer:
-                    self.logger.info(f"{self.name}: {file} file-based syncing")
-                    # todo: complete here the code to not rely on local parser
-                    file_transfer = FileTransfer()
-                    for record in file_transfer.get_file(
-                        self.config, file, self.stream_reader, self.logger
+                    for file_record_data, file_reference in file_transfer.upload(
+                        file=file, stream_reader=self.stream_reader, logger=self.logger
                     ):
-                        line_no += 1
-                        if not self.record_passes_validation_policy(record):
-                            n_skipped += 1
-                            continue
-                        record = self.transform_record_for_file_transfer(record, file)
                         yield stream_data_to_airbyte_message(
-                            self.name, record, is_file_transfer_message=True
+                            self.name,
+                            file_record_data.dict(exclude_none=True),
+                            file_reference=file_reference,
                         )
                 else:
                     for record in parser.parse_records(
@@ -259,6 +239,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
 
     @cache
     def get_json_schema(self) -> JsonSchema:
+        if self.use_file_transfer:
+            return file_transfer_schema
         extra_fields = {
             self.ab_last_mod_col: {"type": "string"},
             self.ab_file_name_col: {"type": "string"},
@@ -282,9 +264,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             return {"type": "object", "properties": {**extra_fields, **schema["properties"]}}
 
     def _get_raw_json_schema(self) -> JsonSchema:
-        if self.use_file_transfer:
-            return file_transfer_schema
-        elif self.config.input_schema:
+        if self.config.input_schema:
             return self.config.get_input_schema()  # type: ignore
         elif self.config.schemaless:
             return schemaless_schema
@@ -341,6 +321,11 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             self.config.globs or [], self.config.legacy_prefix, self.logger
         )
 
+    def as_airbyte_stream(self) -> AirbyteStream:
+        file_stream = super().as_airbyte_stream()
+        file_stream.is_file_based = self.use_file_transfer
+        return file_stream
+
     def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]:
         loop = asyncio.get_event_loop()
         schema = loop.run_until_complete(self._infer_schema(files))