airbyte-cdk 6.45.1.post47.dev14456468218__tar.gz → 6.45.2.dev0__tar.gz

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.45.1.post47.dev14456468218
+Version: 6.45.2.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -22,7 +22,7 @@ Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: airbyte-protocol-models-dataclasses (>=0.15,<0.16)
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
 Requires-Dist: anyascii (>=0.3.2,<0.4.0)
 Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
 Requires-Dist: backoff

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/models/__init__.py

@@ -19,7 +19,6 @@ from .airbyte_protocol import (
     AirbyteMessage,
     AirbyteProtocol,
     AirbyteRecordMessage,
-    AirbyteRecordMessageFileReference,
     AirbyteStateBlob,
     AirbyteStateMessage,
     AirbyteStateStats,

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/models/airbyte_protocol.py

@@ -8,6 +8,8 @@ from typing import Annotated, Any, Dict, List, Mapping, Optional, Union
 from airbyte_protocol_dataclasses.models import *  # noqa: F403  # Allow '*'
 from serpyco_rs.metadata import Alias
 
+from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
+
 # ruff: noqa: F405  # ignore fuzzy import issues with 'import *'
 
 
@@ -82,7 +84,7 @@ class AirbyteMessage:
     spec: Optional[ConnectorSpecification] = None  # type: ignore [name-defined]
     connectionStatus: Optional[AirbyteConnectionStatus] = None  # type: ignore [name-defined]
     catalog: Optional[AirbyteCatalog] = None  # type: ignore [name-defined]
-    record: Optional[AirbyteRecordMessage] = None  # type: ignore [name-defined]
+    record: Optional[Union[AirbyteFileTransferRecordMessage, AirbyteRecordMessage]] = None  # type: ignore [name-defined]
     state: Optional[AirbyteStateMessage] = None
     trace: Optional[AirbyteTraceMessage] = None  # type: ignore [name-defined]
     control: Optional[AirbyteControlMessage] = None  # type: ignore [name-defined]

airbyte_cdk-6.45.2.dev0/airbyte_cdk/models/file_transfer_record_message.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+
+@dataclass
+class AirbyteFileTransferRecordMessage:
+    stream: str
+    file: Dict[str, Any]
+    emitted_at: int
+    namespace: Optional[str] = None
+    data: Optional[Dict[str, Any]] = None

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py

@@ -149,7 +149,7 @@ class ConcurrentReadProcessor:
         message = stream_data_to_airbyte_message(
             stream_name=record.stream_name,
             data_or_message=record.data,
-            file_reference=record.file_reference,
+            is_file_transfer_message=record.is_file_transfer_message,
         )
         stream = self._stream_name_to_instance[record.stream_name]
 

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -25,7 +25,6 @@ from airbyte_cdk.sources.declarative.incremental.per_partition_with_global impor
     PerPartitionWithGlobalCursor,
 )
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
-from airbyte_cdk.sources.declarative.models import FileUploader
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ConcurrencyLevel as ConcurrencyLevelModel,
 )
@@ -207,10 +206,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             # these legacy Python streams the way we do low-code streams to determine if they are concurrent compatible,
             # so we need to treat them as synchronous
 
-            supports_file_transfer = (
-                "file_uploader" in name_to_stream_mapping[declarative_stream.name]
-            )
-
             if (
                 isinstance(declarative_stream, DeclarativeStream)
                 and name_to_stream_mapping[declarative_stream.name]["type"]
@@ -327,7 +322,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             else None,
                             logger=self.logger,
                             cursor=cursor,
-                            supports_file_transfer=supports_file_transfer,
                         )
                     )
                 elif (
@@ -359,7 +353,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             cursor_field=None,
                             logger=self.logger,
                             cursor=final_state_cursor,
-                            supports_file_transfer=supports_file_transfer,
                         )
                     )
                 elif (
@@ -413,7 +406,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             cursor_field=perpartition_cursor.cursor_field.cursor_field_key,
                             logger=self.logger,
                             cursor=perpartition_cursor,
-                            supports_file_transfer=supports_file_transfer,
                         )
                     )
                 else:

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1448,42 +1448,6 @@ definitions:
             - "$ref": "#/definitions/LegacyToPerPartitionStateMigration"
             - "$ref": "#/definitions/CustomStateMigration"
         default: []
-      file_uploader:
-        title: File Uploader
-        description: (experimental) Describes how to fetch a file
-        type: object
-        required:
-          - type
-          - requester
-          - download_target_extractor
-        properties:
-          type:
-            type: string
-            enum: [ FileUploader ]
-          requester:
-            description: Requester component that describes how to prepare HTTP requests to send to the source API.
-            anyOf:
-              - "$ref": "#/definitions/CustomRequester"
-              - "$ref": "#/definitions/HttpRequester"
-          download_target_extractor:
-            description: Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response
-            anyOf:
-              - "$ref": "#/definitions/CustomRecordExtractor"
-              - "$ref": "#/definitions/DpathExtractor"
-          file_extractor:
-            description: Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content
-            anyOf:
-              - "$ref": "#/definitions/CustomRecordExtractor"
-              - "$ref": "#/definitions/DpathExtractor"
-          filename_extractor:
-            description: Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.
-            type: string
-            interpolation_context:
-              - config
-              - record
-            examples:
-              - "{{ record.id }}/{{ record.file_name }}/"
-              - "{{ record.id }}_{{ record.file_name }}/"
       $parameters:
         type: object
         additional_properties: true

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/declarative/extractors/record_selector.py

@@ -15,7 +15,6 @@ from airbyte_cdk.sources.declarative.extractors.type_transformer import (
 )
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.models import SchemaNormalization
-from airbyte_cdk.sources.declarative.retrievers.file_uploader import FileUploader
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
 from airbyte_cdk.sources.utils.transform import TypeTransformer
@@ -43,7 +42,6 @@ class RecordSelector(HttpSelector):
     record_filter: Optional[RecordFilter] = None
     transformations: List[RecordTransformation] = field(default_factory=lambda: [])
     transform_before_filtering: bool = False
-    file_uploader: Optional[FileUploader] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters
@@ -119,10 +117,7 @@ class RecordSelector(HttpSelector):
             transformed_filtered_data, schema=records_schema
         )
         for data in normalized_data:
-            record = Record(data=data, stream_name=self.name, associated_slice=stream_slice)
-            if self.file_uploader:
-                self.file_uploader.upload(record)
-            yield record
+            yield Record(data=data, stream_name=self.name, associated_slice=stream_slice)
 
     def _normalize_by_schema(
         self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -2042,31 +2042,6 @@ class SelectiveAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class FileUploader(BaseModel):
-    type: Literal["FileUploader"]
-    requester: Union[CustomRequester, HttpRequester] = Field(
-        ...,
-        description="Requester component that describes how to prepare HTTP requests to send to the source API.",
-    )
-    download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
-        ...,
-        description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response",
-    )
-    file_extractor: Optional[Union[CustomRecordExtractor, DpathExtractor]] = Field(
-        None,
-        description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content",
-    )
-    filename_extractor: Optional[str] = Field(
-        None,
-        description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.",
-        examples=[
-            "{{ record.id }}/{{ record.file_name }}/",
-            "{{ record.id }}_{{ record.file_name }}/",
-        ],
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class DeclarativeStream(BaseModel):
     class Config:
         extra = Extra.allow
@@ -2125,11 +2100,6 @@ class DeclarativeStream(BaseModel):
         description="Array of state migrations to be applied on the input state",
         title="State Migrations",
     )
-    file_uploader: Optional[FileUploader] = Field(
-        None,
-        description="(experimental) Describes how to fetch a file",
-        title="File Uploader",
-    )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
@@ -2623,7 +2593,6 @@ CompositeErrorHandler.update_forward_refs()
 DeclarativeSource1.update_forward_refs()
 DeclarativeSource2.update_forward_refs()
 SelectiveAuthenticator.update_forward_refs()
-FileUploader.update_forward_refs()
 DeclarativeStream.update_forward_refs()
 SessionTokenAuthenticator.update_forward_refs()
 DynamicSchemaLoader.update_forward_refs()

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -106,6 +106,7 @@ from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_mi
 )
 from airbyte_cdk.sources.declarative.models import (
     CustomStateMigration,
+    GzipDecoder,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     AddedFieldDefinition as AddedFieldDefinitionModel,
@@ -227,9 +228,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    FileUploader as FileUploaderModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel,
 )
@@ -481,7 +479,6 @@ from airbyte_cdk.sources.declarative.retrievers import (
     SimpleRetriever,
     SimpleRetrieverTestReadDecorator,
 )
-from airbyte_cdk.sources.declarative.retrievers.file_uploader import FileUploader
 from airbyte_cdk.sources.declarative.schema import (
     ComplexFieldType,
     DefaultSchemaLoader,
@@ -678,7 +675,6 @@ class ModelToComponentFactory:
             ComponentMappingDefinitionModel: self.create_components_mapping_definition,
             ZipfileDecoderModel: self.create_zipfile_decoder,
             HTTPAPIBudgetModel: self.create_http_api_budget,
-            FileUploaderModel: self.create_file_uploader,
             FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy,
             MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy,
             UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
@@ -1830,11 +1826,6 @@ class ModelToComponentFactory:
                 transformations.append(
                     self._create_component_from_model(model=transformation_model, config=config)
                 )
-        file_uploader = None
-        if model.file_uploader:
-            file_uploader = self._create_component_from_model(
-                model=model.file_uploader, config=config
-            )
 
         retriever = self._create_component_from_model(
             model=model.retriever,
@@ -1846,7 +1837,6 @@ class ModelToComponentFactory:
             stop_condition_on_cursor=stop_condition_on_cursor,
             client_side_incremental_sync=client_side_incremental_sync,
             transformations=transformations,
-            file_uploader=file_uploader,
             incremental_sync=model.incremental_sync,
         )
         cursor_field = model.incremental_sync.cursor_field if model.incremental_sync else None
@@ -2064,6 +2054,7 @@ class ModelToComponentFactory:
         config: Config,
         *,
         url_base: str,
+        extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None,
         decoder: Optional[Decoder] = None,
         cursor_used_for_stop_condition: Optional[DeclarativeCursor] = None,
     ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]:
@@ -2085,7 +2076,10 @@ class ModelToComponentFactory:
             else None
         )
         pagination_strategy = self._create_component_from_model(
-            model=model.pagination_strategy, config=config, decoder=decoder_to_use
+            model=model.pagination_strategy,
+            config=config,
+            decoder=decoder_to_use,
+            extractor_model=extractor_model,
         )
         if cursor_used_for_stop_condition:
             pagination_strategy = StopConditionPaginationStrategyDecorator(
@@ -2582,7 +2576,12 @@ class ModelToComponentFactory:
         )
 
     def create_offset_increment(
-        self, model: OffsetIncrementModel, config: Config, decoder: Decoder, **kwargs: Any
+        self,
+        model: OffsetIncrementModel,
+        config: Config,
+        decoder: Decoder,
+        extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None,
+        **kwargs: Any,
     ) -> OffsetIncrement:
         if isinstance(decoder, PaginationDecoderDecorator):
             inner_decoder = decoder.decoder
@@ -2597,10 +2596,24 @@ class ModelToComponentFactory:
                 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder))
             )
 
+        # Ideally we would instantiate the runtime extractor from highest most level (in this case the SimpleRetriever)
+        # so that it can be shared by OffSetIncrement and RecordSelector. However, due to how we instantiate the
+        # decoder with various decorators here, but not in create_record_selector, it is simpler to retain existing
+        # behavior by having two separate extractors with identical behavior since they use the same extractor model.
+        # When we have more time to investigate we can look into reusing the same component.
+        extractor = (
+            self._create_component_from_model(
+                model=extractor_model, config=config, decoder=decoder_to_use
+            )
+            if extractor_model
+            else None
+        )
+
         return OffsetIncrement(
             page_size=model.page_size,
             config=config,
             decoder=decoder_to_use,
+            extractor=extractor,
             inject_on_first_request=model.inject_on_first_request or False,
             parameters=model.parameters or {},
         )
@@ -2769,7 +2782,6 @@ class ModelToComponentFactory:
         transformations: List[RecordTransformation] | None = None,
         decoder: Decoder | None = None,
         client_side_incremental_sync: Dict[str, Any] | None = None,
-        file_uploader: Optional[FileUploader] = None,
         **kwargs: Any,
     ) -> RecordSelector:
         extractor = self._create_component_from_model(
@@ -2807,7 +2819,6 @@ class ModelToComponentFactory:
             config=config,
             record_filter=record_filter,
             transformations=transformations or [],
-            file_uploader=file_uploader,
             schema_normalization=schema_normalization,
             parameters=model.parameters or {},
             transform_before_filtering=transform_before_filtering,
@@ -2865,7 +2876,6 @@ class ModelToComponentFactory:
         stop_condition_on_cursor: bool = False,
         client_side_incremental_sync: Optional[Dict[str, Any]] = None,
         transformations: List[RecordTransformation],
-        file_uploader: Optional[FileUploader] = None,
         incremental_sync: Optional[
             Union[
                 IncrementingCountCursorModel, DatetimeBasedCursorModel, CustomIncrementalSyncModel
@@ -2886,7 +2896,6 @@ class ModelToComponentFactory:
             decoder=decoder,
             transformations=transformations,
             client_side_incremental_sync=client_side_incremental_sync,
-            file_uploader=file_uploader,
         )
 
         query_properties: Optional[QueryProperties] = None
@@ -2968,6 +2977,7 @@ class ModelToComponentFactory:
                 model=model.paginator,
                 config=config,
                 url_base=url_base,
+                extractor_model=model.record_selector.extractor,
                 decoder=decoder,
                 cursor_used_for_stop_condition=cursor_used_for_stop_condition,
             )
@@ -3552,30 +3562,6 @@ class ModelToComponentFactory:
             matchers=matchers,
         )
 
-    def create_file_uploader(
-        self, model: FileUploaderModel, config: Config, **kwargs: Any
-    ) -> FileUploader:
-        name = "File Uploader"
-        requester = self._create_component_from_model(
-            model=model.requester,
-            config=config,
-            name=name,
-            **kwargs,
-        )
-        download_target_extractor = self._create_component_from_model(
-            model=model.download_target_extractor,
-            config=config,
-            name=name,
-            **kwargs,
-        )
-        return FileUploader(
-            requester=requester,
-            download_target_extractor=download_target_extractor,
-            config=config,
-            parameters=model.parameters or {},
-            filename_extractor=model.filename_extractor if model.filename_extractor else None,
-        )
-
     def create_moving_window_call_rate_policy(
         self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any
     ) -> MovingWindowCallRatePolicy:

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py

@@ -12,6 +12,7 @@ from airbyte_cdk.sources.declarative.decoders import (
     JsonDecoder,
     PaginationDecoderDecorator,
 )
+from airbyte_cdk.sources.declarative.extractors.dpath_extractor import RecordExtractor
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
     PaginationStrategy,
@@ -46,6 +47,7 @@ class OffsetIncrement(PaginationStrategy):
     config: Config
     page_size: Optional[Union[str, int]]
     parameters: InitVar[Mapping[str, Any]]
+    extractor: Optional[RecordExtractor]
     decoder: Decoder = field(
         default_factory=lambda: PaginationDecoderDecorator(decoder=JsonDecoder(parameters={}))
     )
@@ -75,6 +77,14 @@ class OffsetIncrement(PaginationStrategy):
     ) -> Optional[Any]:
         decoded_response = next(self.decoder.decode(response))
 
+        if self.extractor:
+            page_size_from_response = len(list(self.extractor.extract_records(response=response)))
+            # The extractor could return 0 records which is valid, but evaluates to False. Our fallback in other
+            # cases as the best effort option is to use the incoming last_page_size
+            last_page_size = (
+                page_size_from_response if page_size_from_response is not None else last_page_size
+            )
+
         # Stop paginating when there are fewer records than the page size or the current page has no records
         if (
             self._page_size

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -364,7 +364,7 @@ class SimpleRetriever(Retriever):
         pagination_complete = False
         initial_token = self._paginator.get_initial_token()
         next_page_token: Optional[Mapping[str, Any]] = (
-            {"next_page_token": initial_token} if initial_token else None
+            {"next_page_token": initial_token} if initial_token is not None else None
         )
         while not pagination_complete:
             response = self._fetch_next_page(stream_state, stream_slice, next_page_token)

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -58,16 +58,11 @@ class DeclarativePartition(Partition):
     def read(self) -> Iterable[Record]:
         for stream_data in self._retriever.read_records(self._json_schema, self._stream_slice):
             if isinstance(stream_data, Mapping):
-                record = (
-                    stream_data
-                    if isinstance(stream_data, Record)
-                    else Record(
-                        data=stream_data,
-                        stream_name=self.stream_name(),
-                        associated_slice=self._stream_slice,
-                    )
+                yield Record(
+                    data=stream_data,
+                    stream_name=self.stream_name(),
+                    associated_slice=self._stream_slice,
                 )
-                yield record
             else:
                 self._message_repository.emit_message(stream_data)
 

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -8,18 +8,16 @@ from datetime import datetime
 from enum import Enum
 from io import IOBase
 from os import makedirs, path
-from typing import Any, Callable, Iterable, List, MutableMapping, Optional, Set, Tuple
+from typing import Any, Dict, Iterable, List, Optional, Set
 
 from wcmatch.glob import GLOBSTAR, globmatch
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
 from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import (
     include_identities_stream,
     preserve_directory_structure,
     use_file_transfer,
 )
-from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 
 
@@ -30,10 +28,6 @@ class FileReadMode(Enum):
 
 class AbstractFileBasedStreamReader(ABC):
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
-    FILE_RELATIVE_PATH = "file_relative_path"
-    FILE_NAME = "file_name"
-    LOCAL_FILE_PATH = "local_file_path"
-    FILE_FOLDER = "file_folder"
 
     def __init__(self) -> None:
         self._config = None
@@ -154,9 +148,9 @@ class AbstractFileBasedStreamReader(ABC):
         return False
 
     @abstractmethod
-    def upload(
+    def get_file(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
-    ) -> Tuple[FileRecordData, AirbyteRecordMessageFileReference]:
+    ) -> Dict[str, Any]:
         """
         This is required for connectors that will support writing to
         files. It will handle the logic to download,get,read,acquire or
@@ -168,41 +162,25 @@ class AbstractFileBasedStreamReader(ABC):
                logger (logging.Logger): Logger for logging information and errors.
 
            Returns:
-               AirbyteRecordMessageFileReference: A file reference object containing:
-                   - staging_file_url (str): The absolute path to the referenced file in the staging area.
-                   - file_size_bytes (int): The size of the referenced file in bytes.
-                   - source_file_relative_path (str): The relative path to the referenced file in source.
+               dict: A dictionary containing the following:
+                   - "file_url" (str): The absolute path of the downloaded file.
+                   - "bytes" (int): The file size in bytes.
+                   - "file_relative_path" (str): The relative path of the file for local storage. Is relative to local_directory as
+                   this a mounted volume in the pod container.
+
         """
         ...
 
-    def _get_file_transfer_paths(
-        self, source_file_relative_path: str, staging_directory: str
-    ) -> MutableMapping[str, Any]:
-        """
-        This method is used to get the file transfer paths for a given source file relative path and local directory.
-        It returns a dictionary with the following keys:
-            - FILE_RELATIVE_PATH: The relative path to file in reference to the staging directory.
-            - LOCAL_FILE_PATH: The absolute path to the file.
-            - FILE_NAME: The name of the referenced file.
-            - FILE_FOLDER: The folder of the referenced file.
-        """
+    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
         preserve_directory_structure = self.preserve_directory_structure()
-
-        file_name = path.basename(source_file_relative_path)
-        file_folder = path.dirname(source_file_relative_path)
         if preserve_directory_structure:
             # Remove left slashes from source path format to make relative path for writing locally
-            file_relative_path = source_file_relative_path.lstrip("/")
+            file_relative_path = file.uri.lstrip("/")
         else:
-            file_relative_path = file_name
-        local_file_path = path.join(staging_directory, file_relative_path)
+            file_relative_path = path.basename(file.uri)
+        local_file_path = path.join(local_directory, file_relative_path)
+
         # Ensure the local directory exists
         makedirs(path.dirname(local_file_path), exist_ok=True)
-
-        file_paths = {
-            self.FILE_RELATIVE_PATH: file_relative_path,
-            self.LOCAL_FILE_PATH: local_file_path,
-            self.FILE_NAME: file_name,
-            self.FILE_FOLDER: file_folder,
-        }
-        return file_paths
+        absolute_file_path = path.abspath(local_file_path)
+        return [file_relative_path, local_file_path, absolute_file_path]

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/file_based/file_types/file_transfer.py

@@ -2,27 +2,34 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 #
 import logging
-from typing import Iterable, Tuple
+import os
+from typing import Any, Dict, Iterable
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
-from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.utils.files_directory import get_files_directory
+
+AIRBYTE_STAGING_DIRECTORY = os.getenv("AIRBYTE_STAGING_DIRECTORY", "/staging/files")
+DEFAULT_LOCAL_DIRECTORY = "/tmp/airbyte-file-transfer"
 
 
 class FileTransfer:
     def __init__(self) -> None:
-        self._local_directory = get_files_directory()
+        self._local_directory = (
+            AIRBYTE_STAGING_DIRECTORY
+            if os.path.exists(AIRBYTE_STAGING_DIRECTORY)
+            else DEFAULT_LOCAL_DIRECTORY
+        )
 
-    def upload(
+    def get_file(
         self,
+        config: FileBasedStreamConfig,
         file: RemoteFile,
         stream_reader: AbstractFileBasedStreamReader,
         logger: logging.Logger,
-    ) -> Iterable[Tuple[FileRecordData, AirbyteRecordMessageFileReference]]:
+    ) -> Iterable[Dict[str, Any]]:
         try:
-            yield stream_reader.upload(
+            yield stream_reader.get_file(
                 file=file, local_directory=self._local_directory, logger=logger
             )
         except Exception as ex:

{airbyte_cdk-6.45.1.post47.dev14456468218 → airbyte_cdk-6.45.2.dev0}/airbyte_cdk/sources/file_based/schema_helpers.py

@@ -18,18 +18,9 @@ JsonSchemaSupportedType = Union[List[str], Literal["string"], str]
 SchemaType = Mapping[str, Mapping[str, JsonSchemaSupportedType]]
 
 schemaless_schema = {"type": "object", "properties": {"data": {"type": "object"}}}
-
 file_transfer_schema = {
     "type": "object",
-    "properties": {
-        "folder": {"type": "string"},
-        "file_name": {"type": "string"},
-        "source_uri": {"type": "string"},
-        "bytes": {"type": "integer"},
-        "id": {"type": ["null", "string"]},
-        "updated_at": {"type": ["null", "string"]},
-        "mime_type": {"type": ["null", "string"]},
-    },
+    "properties": {"data": {"type": "object"}, "file": {"type": "object"}},
 }