airbyte-cdk 6.45.1.post47.dev14456468218__py3-none-any.whl → 6.45.2.post3.dev14463482961__py3-none-any.whl

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -4,7 +4,7 @@
 
 
 from dataclasses import asdict, dataclass, field
-from typing import Any, Dict, List, Mapping
+from typing import Any, ClassVar, Dict, List, Mapping
 
 from airbyte_cdk.connector_builder.test_reader import TestReader
 from airbyte_cdk.models import (
@@ -37,6 +37,8 @@ MAX_STREAMS_KEY = "max_streams"
 
 @dataclass
 class TestLimits:
+    __test__: ClassVar[bool] = False  # Tell Pytest this is not a Pytest class, despite its name
+
     max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS)
     max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE)
     max_slices: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_SLICES)

airbyte_cdk/connector_builder/test_reader/reader.py

@@ -4,7 +4,7 @@
 
 
 import logging
-from typing import Any, Dict, Iterator, List, Mapping, Optional, Union
+from typing import Any, ClassVar, Dict, Iterator, List, Mapping, Optional, Union
 
 from airbyte_cdk.connector_builder.models import (
     AuxiliaryRequest,
@@ -66,6 +66,8 @@ class TestReader:
 
     """
 
+    __test__: ClassVar[bool] = False  # Tell Pytest this is not a Pytest class, despite its name
+
     logger = logging.getLogger("airbyte.connector-builder")
 
     def __init__(

airbyte_cdk/models/__init__.py

@@ -19,7 +19,6 @@ from .airbyte_protocol import (
     AirbyteMessage,
     AirbyteProtocol,
     AirbyteRecordMessage,
-    AirbyteRecordMessageFileReference,
     AirbyteStateBlob,
     AirbyteStateMessage,
     AirbyteStateStats,

airbyte_cdk/models/airbyte_protocol.py

@@ -8,6 +8,8 @@ from typing import Annotated, Any, Dict, List, Mapping, Optional, Union
 from airbyte_protocol_dataclasses.models import *  # noqa: F403  # Allow '*'
 from serpyco_rs.metadata import Alias
 
+from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
+
 # ruff: noqa: F405  # ignore fuzzy import issues with 'import *'
 
 
@@ -82,7 +84,7 @@ class AirbyteMessage:
     spec: Optional[ConnectorSpecification] = None  # type: ignore [name-defined]
     connectionStatus: Optional[AirbyteConnectionStatus] = None  # type: ignore [name-defined]
     catalog: Optional[AirbyteCatalog] = None  # type: ignore [name-defined]
-    record: Optional[AirbyteRecordMessage] = None  # type: ignore [name-defined]
+    record: Optional[Union[AirbyteFileTransferRecordMessage, AirbyteRecordMessage]] = None  # type: ignore [name-defined]
     state: Optional[AirbyteStateMessage] = None
     trace: Optional[AirbyteTraceMessage] = None  # type: ignore [name-defined]
     control: Optional[AirbyteControlMessage] = None  # type: ignore [name-defined]

airbyte_cdk/models/file_transfer_record_message.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+
+@dataclass
+class AirbyteFileTransferRecordMessage:
+    stream: str
+    file: Dict[str, Any]
+    emitted_at: int
+    namespace: Optional[str] = None
+    data: Optional[Dict[str, Any]] = None

airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py

@@ -149,7 +149,7 @@ class ConcurrentReadProcessor:
         message = stream_data_to_airbyte_message(
             stream_name=record.stream_name,
             data_or_message=record.data,
-            file_reference=record.file_reference,
+            is_file_transfer_message=record.is_file_transfer_message,
         )
         stream = self._stream_name_to_instance[record.stream_name]
 

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -25,7 +25,6 @@ from airbyte_cdk.sources.declarative.incremental.per_partition_with_global impor
     PerPartitionWithGlobalCursor,
 )
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
-from airbyte_cdk.sources.declarative.models import FileUploader
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ConcurrencyLevel as ConcurrencyLevelModel,
 )
@@ -207,10 +206,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             # these legacy Python streams the way we do low-code streams to determine if they are concurrent compatible,
             # so we need to treat them as synchronous
 
-            supports_file_transfer = (
-                "file_uploader" in name_to_stream_mapping[declarative_stream.name]
-            )
-
             if (
                 isinstance(declarative_stream, DeclarativeStream)
                 and name_to_stream_mapping[declarative_stream.name]["type"]
@@ -327,7 +322,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             else None,
                             logger=self.logger,
                             cursor=cursor,
-                            supports_file_transfer=supports_file_transfer,
                         )
                     )
                 elif (
@@ -359,7 +353,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             cursor_field=None,
                             logger=self.logger,
                             cursor=final_state_cursor,
-                            supports_file_transfer=supports_file_transfer,
                         )
                     )
                 elif (
@@ -413,7 +406,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             cursor_field=perpartition_cursor.cursor_field.cursor_field_key,
                             logger=self.logger,
                             cursor=perpartition_cursor,
-                            supports_file_transfer=supports_file_transfer,
                         )
                     )
                 else:

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1448,42 +1448,6 @@ definitions:
             - "$ref": "#/definitions/LegacyToPerPartitionStateMigration"
             - "$ref": "#/definitions/CustomStateMigration"
         default: []
-      file_uploader:
-        title: File Uploader
-        description: (experimental) Describes how to fetch a file
-        type: object
-        required:
-          - type
-          - requester
-          - download_target_extractor
-        properties:
-          type:
-            type: string
-            enum: [ FileUploader ]
-          requester:
-            description: Requester component that describes how to prepare HTTP requests to send to the source API.
-            anyOf:
-              - "$ref": "#/definitions/CustomRequester"
-              - "$ref": "#/definitions/HttpRequester"
-          download_target_extractor:
-            description: Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response
-            anyOf:
-              - "$ref": "#/definitions/CustomRecordExtractor"
-              - "$ref": "#/definitions/DpathExtractor"
-          file_extractor:
-            description: Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content
-            anyOf:
-              - "$ref": "#/definitions/CustomRecordExtractor"
-              - "$ref": "#/definitions/DpathExtractor"
-          filename_extractor:
-            description: Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.
-            type: string
-            interpolation_context:
-              - config
-              - record
-            examples:
-              - "{{ record.id }}/{{ record.file_name }}/"
-              - "{{ record.id }}_{{ record.file_name }}/"
       $parameters:
         type: object
         additional_properties: true

airbyte_cdk/sources/declarative/extractors/record_selector.py

@@ -15,7 +15,6 @@ from airbyte_cdk.sources.declarative.extractors.type_transformer import (
 )
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.models import SchemaNormalization
-from airbyte_cdk.sources.declarative.retrievers.file_uploader import FileUploader
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
 from airbyte_cdk.sources.utils.transform import TypeTransformer
@@ -43,7 +42,6 @@ class RecordSelector(HttpSelector):
     record_filter: Optional[RecordFilter] = None
     transformations: List[RecordTransformation] = field(default_factory=lambda: [])
     transform_before_filtering: bool = False
-    file_uploader: Optional[FileUploader] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters
@@ -119,10 +117,7 @@ class RecordSelector(HttpSelector):
             transformed_filtered_data, schema=records_schema
         )
         for data in normalized_data:
-            record = Record(data=data, stream_name=self.name, associated_slice=stream_slice)
-            if self.file_uploader:
-                self.file_uploader.upload(record)
-            yield record
+            yield Record(data=data, stream_name=self.name, associated_slice=stream_slice)
 
     def _normalize_by_schema(
         self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -2042,31 +2042,6 @@ class SelectiveAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class FileUploader(BaseModel):
-    type: Literal["FileUploader"]
-    requester: Union[CustomRequester, HttpRequester] = Field(
-        ...,
-        description="Requester component that describes how to prepare HTTP requests to send to the source API.",
-    )
-    download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
-        ...,
-        description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response",
-    )
-    file_extractor: Optional[Union[CustomRecordExtractor, DpathExtractor]] = Field(
-        None,
-        description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content",
-    )
-    filename_extractor: Optional[str] = Field(
-        None,
-        description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.",
-        examples=[
-            "{{ record.id }}/{{ record.file_name }}/",
-            "{{ record.id }}_{{ record.file_name }}/",
-        ],
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class DeclarativeStream(BaseModel):
     class Config:
         extra = Extra.allow
@@ -2125,11 +2100,6 @@ class DeclarativeStream(BaseModel):
         description="Array of state migrations to be applied on the input state",
         title="State Migrations",
     )
-    file_uploader: Optional[FileUploader] = Field(
-        None,
-        description="(experimental) Describes how to fetch a file",
-        title="File Uploader",
-    )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
@@ -2623,7 +2593,6 @@ CompositeErrorHandler.update_forward_refs()
 DeclarativeSource1.update_forward_refs()
 DeclarativeSource2.update_forward_refs()
 SelectiveAuthenticator.update_forward_refs()
-FileUploader.update_forward_refs()
 DeclarativeStream.update_forward_refs()
 SessionTokenAuthenticator.update_forward_refs()
 DynamicSchemaLoader.update_forward_refs()

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -106,6 +106,7 @@ from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_mi
 )
 from airbyte_cdk.sources.declarative.models import (
     CustomStateMigration,
+    GzipDecoder,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     AddedFieldDefinition as AddedFieldDefinitionModel,
@@ -227,9 +228,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    FileUploader as FileUploaderModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     FixedWindowCallRatePolicy as FixedWindowCallRatePolicyModel,
 )
@@ -481,7 +479,6 @@ from airbyte_cdk.sources.declarative.retrievers import (
     SimpleRetriever,
     SimpleRetrieverTestReadDecorator,
 )
-from airbyte_cdk.sources.declarative.retrievers.file_uploader import FileUploader
 from airbyte_cdk.sources.declarative.schema import (
     ComplexFieldType,
     DefaultSchemaLoader,
@@ -678,7 +675,6 @@ class ModelToComponentFactory:
             ComponentMappingDefinitionModel: self.create_components_mapping_definition,
             ZipfileDecoderModel: self.create_zipfile_decoder,
             HTTPAPIBudgetModel: self.create_http_api_budget,
-            FileUploaderModel: self.create_file_uploader,
             FixedWindowCallRatePolicyModel: self.create_fixed_window_call_rate_policy,
             MovingWindowCallRatePolicyModel: self.create_moving_window_call_rate_policy,
             UnlimitedCallRatePolicyModel: self.create_unlimited_call_rate_policy,
@@ -1830,11 +1826,6 @@ class ModelToComponentFactory:
                 transformations.append(
                     self._create_component_from_model(model=transformation_model, config=config)
                 )
-        file_uploader = None
-        if model.file_uploader:
-            file_uploader = self._create_component_from_model(
-                model=model.file_uploader, config=config
-            )
 
         retriever = self._create_component_from_model(
             model=model.retriever,
@@ -1846,7 +1837,6 @@ class ModelToComponentFactory:
             stop_condition_on_cursor=stop_condition_on_cursor,
             client_side_incremental_sync=client_side_incremental_sync,
             transformations=transformations,
-            file_uploader=file_uploader,
             incremental_sync=model.incremental_sync,
         )
         cursor_field = model.incremental_sync.cursor_field if model.incremental_sync else None
@@ -2769,7 +2759,6 @@ class ModelToComponentFactory:
         transformations: List[RecordTransformation] | None = None,
         decoder: Decoder | None = None,
         client_side_incremental_sync: Dict[str, Any] | None = None,
-        file_uploader: Optional[FileUploader] = None,
         **kwargs: Any,
     ) -> RecordSelector:
         extractor = self._create_component_from_model(
@@ -2807,7 +2796,6 @@ class ModelToComponentFactory:
             config=config,
             record_filter=record_filter,
             transformations=transformations or [],
-            file_uploader=file_uploader,
             schema_normalization=schema_normalization,
             parameters=model.parameters or {},
             transform_before_filtering=transform_before_filtering,
@@ -2865,7 +2853,6 @@ class ModelToComponentFactory:
         stop_condition_on_cursor: bool = False,
         client_side_incremental_sync: Optional[Dict[str, Any]] = None,
         transformations: List[RecordTransformation],
-        file_uploader: Optional[FileUploader] = None,
         incremental_sync: Optional[
             Union[
                 IncrementingCountCursorModel, DatetimeBasedCursorModel, CustomIncrementalSyncModel
@@ -2886,7 +2873,6 @@ class ModelToComponentFactory:
             decoder=decoder,
             transformations=transformations,
             client_side_incremental_sync=client_side_incremental_sync,
-            file_uploader=file_uploader,
         )
 
         query_properties: Optional[QueryProperties] = None
@@ -3552,30 +3538,6 @@ class ModelToComponentFactory:
             matchers=matchers,
         )
 
-    def create_file_uploader(
-        self, model: FileUploaderModel, config: Config, **kwargs: Any
-    ) -> FileUploader:
-        name = "File Uploader"
-        requester = self._create_component_from_model(
-            model=model.requester,
-            config=config,
-            name=name,
-            **kwargs,
-        )
-        download_target_extractor = self._create_component_from_model(
-            model=model.download_target_extractor,
-            config=config,
-            name=name,
-            **kwargs,
-        )
-        return FileUploader(
-            requester=requester,
-            download_target_extractor=download_target_extractor,
-            config=config,
-            parameters=model.parameters or {},
-            filename_extractor=model.filename_extractor if model.filename_extractor else None,
-        )
-
     def create_moving_window_call_rate_policy(
         self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any
     ) -> MovingWindowCallRatePolicy:

airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -364,7 +364,7 @@ class SimpleRetriever(Retriever):
         pagination_complete = False
         initial_token = self._paginator.get_initial_token()
         next_page_token: Optional[Mapping[str, Any]] = (
-            {"next_page_token": initial_token} if initial_token else None
+            {"next_page_token": initial_token} if initial_token is not None else None
         )
         while not pagination_complete:
             response = self._fetch_next_page(stream_state, stream_slice, next_page_token)

airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -58,16 +58,11 @@ class DeclarativePartition(Partition):
     def read(self) -> Iterable[Record]:
         for stream_data in self._retriever.read_records(self._json_schema, self._stream_slice):
             if isinstance(stream_data, Mapping):
-                record = (
-                    stream_data
-                    if isinstance(stream_data, Record)
-                    else Record(
-                        data=stream_data,
-                        stream_name=self.stream_name(),
-                        associated_slice=self._stream_slice,
-                    )
+                yield Record(
+                    data=stream_data,
+                    stream_name=self.stream_name(),
+                    associated_slice=self._stream_slice,
                 )
-                yield record
             else:
                 self._message_repository.emit_message(stream_data)
 

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -8,18 +8,16 @@ from datetime import datetime
 from enum import Enum
 from io import IOBase
 from os import makedirs, path
-from typing import Any, Callable, Iterable, List, MutableMapping, Optional, Set, Tuple
+from typing import Any, Dict, Iterable, List, Optional, Set
 
 from wcmatch.glob import GLOBSTAR, globmatch
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
 from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import (
     include_identities_stream,
     preserve_directory_structure,
     use_file_transfer,
 )
-from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 
 
@@ -30,10 +28,6 @@ class FileReadMode(Enum):
 
 class AbstractFileBasedStreamReader(ABC):
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
-    FILE_RELATIVE_PATH = "file_relative_path"
-    FILE_NAME = "file_name"
-    LOCAL_FILE_PATH = "local_file_path"
-    FILE_FOLDER = "file_folder"
 
     def __init__(self) -> None:
         self._config = None
@@ -154,9 +148,9 @@ class AbstractFileBasedStreamReader(ABC):
         return False
 
     @abstractmethod
-    def upload(
+    def get_file(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
-    ) -> Tuple[FileRecordData, AirbyteRecordMessageFileReference]:
+    ) -> Dict[str, Any]:
         """
         This is required for connectors that will support writing to
         files. It will handle the logic to download,get,read,acquire or
@@ -168,41 +162,25 @@ class AbstractFileBasedStreamReader(ABC):
                logger (logging.Logger): Logger for logging information and errors.
 
            Returns:
-               AirbyteRecordMessageFileReference: A file reference object containing:
-                   - staging_file_url (str): The absolute path to the referenced file in the staging area.
-                   - file_size_bytes (int): The size of the referenced file in bytes.
-                   - source_file_relative_path (str): The relative path to the referenced file in source.
+               dict: A dictionary containing the following:
+                   - "file_url" (str): The absolute path of the downloaded file.
+                   - "bytes" (int): The file size in bytes.
+                   - "file_relative_path" (str): The relative path of the file for local storage. Is relative to local_directory as
+                   this a mounted volume in the pod container.
+
         """
         ...
 
-    def _get_file_transfer_paths(
-        self, source_file_relative_path: str, staging_directory: str
-    ) -> MutableMapping[str, Any]:
-        """
-        This method is used to get the file transfer paths for a given source file relative path and local directory.
-        It returns a dictionary with the following keys:
-            - FILE_RELATIVE_PATH: The relative path to file in reference to the staging directory.
-            - LOCAL_FILE_PATH: The absolute path to the file.
-            - FILE_NAME: The name of the referenced file.
-            - FILE_FOLDER: The folder of the referenced file.
-        """
+    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
         preserve_directory_structure = self.preserve_directory_structure()
-
-        file_name = path.basename(source_file_relative_path)
-        file_folder = path.dirname(source_file_relative_path)
         if preserve_directory_structure:
             # Remove left slashes from source path format to make relative path for writing locally
-            file_relative_path = source_file_relative_path.lstrip("/")
+            file_relative_path = file.uri.lstrip("/")
         else:
-            file_relative_path = file_name
-        local_file_path = path.join(staging_directory, file_relative_path)
+            file_relative_path = path.basename(file.uri)
+        local_file_path = path.join(local_directory, file_relative_path)
+
         # Ensure the local directory exists
         makedirs(path.dirname(local_file_path), exist_ok=True)
-
-        file_paths = {
-            self.FILE_RELATIVE_PATH: file_relative_path,
-            self.LOCAL_FILE_PATH: local_file_path,
-            self.FILE_NAME: file_name,
-            self.FILE_FOLDER: file_folder,
-        }
-        return file_paths
+        absolute_file_path = path.abspath(local_file_path)
+        return [file_relative_path, local_file_path, absolute_file_path]

airbyte_cdk/sources/file_based/file_types/file_transfer.py

@@ -2,27 +2,34 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 #
 import logging
-from typing import Iterable, Tuple
+import os
+from typing import Any, Dict, Iterable
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
-from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.utils.files_directory import get_files_directory
+
+AIRBYTE_STAGING_DIRECTORY = os.getenv("AIRBYTE_STAGING_DIRECTORY", "/staging/files")
+DEFAULT_LOCAL_DIRECTORY = "/tmp/airbyte-file-transfer"
 
 
 class FileTransfer:
     def __init__(self) -> None:
-        self._local_directory = get_files_directory()
+        self._local_directory = (
+            AIRBYTE_STAGING_DIRECTORY
+            if os.path.exists(AIRBYTE_STAGING_DIRECTORY)
+            else DEFAULT_LOCAL_DIRECTORY
+        )
 
-    def upload(
+    def get_file(
         self,
+        config: FileBasedStreamConfig,
         file: RemoteFile,
         stream_reader: AbstractFileBasedStreamReader,
         logger: logging.Logger,
-    ) -> Iterable[Tuple[FileRecordData, AirbyteRecordMessageFileReference]]:
+    ) -> Iterable[Dict[str, Any]]:
         try:
-            yield stream_reader.upload(
+            yield stream_reader.get_file(
                 file=file, local_directory=self._local_directory, logger=logger
             )
         except Exception as ex:

airbyte_cdk/sources/file_based/schema_helpers.py

@@ -18,18 +18,9 @@ JsonSchemaSupportedType = Union[List[str], Literal["string"], str]
 SchemaType = Mapping[str, Mapping[str, JsonSchemaSupportedType]]
 
 schemaless_schema = {"type": "object", "properties": {"data": {"type": "object"}}}
-
 file_transfer_schema = {
     "type": "object",
-    "properties": {
-        "folder": {"type": "string"},
-        "file_name": {"type": "string"},
-        "source_uri": {"type": "string"},
-        "bytes": {"type": "integer"},
-        "id": {"type": ["null", "string"]},
-        "updated_at": {"type": ["null", "string"]},
-        "mime_type": {"type": ["null", "string"]},
-    },
+    "properties": {"data": {"type": "object"}, "file": {"type": "object"}},
 }
 
 

airbyte_cdk/sources/file_based/stream/concurrent/adapters.py

@@ -4,7 +4,7 @@
 
 import copy
 import logging
-from functools import lru_cache
+from functools import cache, lru_cache
 from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
 from typing_extensions import deprecated
@@ -258,14 +258,19 @@ class FileBasedStreamPartition(Partition):
                     and record_data.record is not None
                 ):
                     # `AirbyteMessage`s of type `Record` should also be yielded so they are enqueued
-                    record_message_data = record_data.record.data
+                    # If stream is flagged for file_transfer the record should data in file key
+                    record_message_data = (
+                        record_data.record.file
+                        if self._use_file_transfer()
+                        else record_data.record.data
+                    )
                     if not record_message_data:
                         raise ExceptionWithDisplayMessage("A record without data was found")
                     else:
                         yield Record(
                             data=record_message_data,
                             stream_name=self.stream_name(),
-                            file_reference=record_data.record.file_reference,
+                            is_file_transfer_message=self._use_file_transfer(),
                         )
                 else:
                     self._message_repository.emit_message(record_data)
@@ -301,6 +306,10 @@ class FileBasedStreamPartition(Partition):
     def stream_name(self) -> str:
         return self._stream.name
 
+    @cache
+    def _use_file_transfer(self) -> bool:
+        return hasattr(self._stream, "use_file_transfer") and self._stream.use_file_transfer
+
     def __repr__(self) -> str:
         return f"FileBasedStreamPartition({self._stream.name}, {self._slice})"
 

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -11,7 +11,7 @@ from functools import cache
 from os import path
 from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteStream, FailureType, Level
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.exceptions import (
@@ -56,7 +56,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
     use_file_transfer = False
     preserve_directory_structure = True
-    _file_transfer = FileTransfer()
 
     def __init__(self, **kwargs: Any):
         if self.FILE_TRANSFER_KW in kwargs:
@@ -94,6 +93,21 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             self.config
         )
 
+    def _filter_schema_invalid_properties(
+        self, configured_catalog_json_schema: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        if self.use_file_transfer:
+            return {
+                "type": "object",
+                "properties": {
+                    "file_path": {"type": "string"},
+                    "file_size": {"type": "string"},
+                    self.ab_file_name_col: {"type": "string"},
+                },
+            }
+        else:
+            return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
+
     def _duplicated_files_names(
         self, slices: List[dict[str, List[RemoteFile]]]
     ) -> List[dict[str, List[str]]]:
@@ -131,6 +145,14 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         record[self.ab_file_name_col] = file.uri
         return record
 
+    def transform_record_for_file_transfer(
+        self, record: dict[str, Any], file: RemoteFile
+    ) -> dict[str, Any]:
+        # timstamp() returns a float representing the number of seconds since the unix epoch
+        record[self.modified] = int(file.last_modified.timestamp()) * 1000
+        record[self.source_file_url] = file.uri
+        return record
+
     def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
         """
         Yield all records from all remote files in `list_files_for_this_sync`.
@@ -151,13 +173,19 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
 
             try:
                 if self.use_file_transfer:
-                    for file_record_data, file_reference in self._file_transfer.upload(
-                        file=file, stream_reader=self.stream_reader, logger=self.logger
+                    self.logger.info(f"{self.name}: {file} file-based syncing")
+                    # todo: complete here the code to not rely on local parser
+                    file_transfer = FileTransfer()
+                    for record in file_transfer.get_file(
+                        self.config, file, self.stream_reader, self.logger
                     ):
+                        line_no += 1
+                        if not self.record_passes_validation_policy(record):
+                            n_skipped += 1
+                            continue
+                        record = self.transform_record_for_file_transfer(record, file)
                         yield stream_data_to_airbyte_message(
-                            self.name,
-                            file_record_data.dict(exclude_none=True),
-                            file_reference=file_reference,
+                            self.name, record, is_file_transfer_message=True
                         )
                 else:
                     for record in parser.parse_records(
@@ -231,8 +259,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
 
     @cache
     def get_json_schema(self) -> JsonSchema:
-        if self.use_file_transfer:
-            return file_transfer_schema
         extra_fields = {
             self.ab_last_mod_col: {"type": "string"},
             self.ab_file_name_col: {"type": "string"},
@@ -256,7 +282,9 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             return {"type": "object", "properties": {**extra_fields, **schema["properties"]}}
 
     def _get_raw_json_schema(self) -> JsonSchema:
-        if self.config.input_schema:
+        if self.use_file_transfer:
+            return file_transfer_schema
+        elif self.config.input_schema:
             return self.config.get_input_schema()  # type: ignore
         elif self.config.schemaless:
             return schemaless_schema
@@ -313,11 +341,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
             self.config.globs or [], self.config.legacy_prefix, self.logger
         )
 
-    def as_airbyte_stream(self) -> AirbyteStream:
-        file_stream = super().as_airbyte_stream()
-        file_stream.is_file_based = self.use_file_transfer
-        return file_stream
-
     def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]:
         loop = asyncio.get_event_loop()
         schema = loop.run_until_complete(self._infer_schema(files))

airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py

@@ -61,7 +61,9 @@ class PermissionsFileBasedStream(DefaultFileBasedStream):
                 permissions_record = self.transform_record(
                     permissions_record, file, file_datetime_string
                 )
-                yield stream_data_to_airbyte_message(self.name, permissions_record)
+                yield stream_data_to_airbyte_message(
+                    self.name, permissions_record, is_file_transfer_message=False
+                )
             except Exception as e:
                 self.logger.error(f"Failed to retrieve permissions for file {file.uri}: {str(e)}")
                 yield AirbyteMessage(

airbyte_cdk/sources/streams/concurrent/default_stream.py

@@ -29,7 +29,6 @@ class DefaultStream(AbstractStream):
         logger: Logger,
         cursor: Cursor,
         namespace: Optional[str] = None,
-        supports_file_transfer: bool = False,
     ) -> None:
         self._stream_partition_generator = partition_generator
         self._name = name
@@ -40,7 +39,6 @@ class DefaultStream(AbstractStream):
         self._logger = logger
         self._cursor = cursor
         self._namespace = namespace
-        self._supports_file_transfer = supports_file_transfer
 
     def generate_partitions(self) -> Iterable[Partition]:
         yield from self._stream_partition_generator.generate()
@@ -70,7 +68,6 @@ class DefaultStream(AbstractStream):
             json_schema=dict(self._json_schema),
             supported_sync_modes=[SyncMode.full_refresh],
             is_resumable=False,
-            is_file_based=self._supports_file_transfer,
         )
 
         if self._namespace:

airbyte_cdk/sources/types.py

@@ -6,7 +6,6 @@ from __future__ import annotations
 
 from typing import Any, ItemsView, Iterator, KeysView, List, Mapping, Optional, ValuesView
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.utils.slice_hasher import SliceHasher
 
 # A FieldPointer designates a path to a field inside a mapping. For example, retrieving ["k1", "k1.2"] in the object {"k1" :{"k1.2":
@@ -24,12 +23,12 @@ class Record(Mapping[str, Any]):
         data: Mapping[str, Any],
         stream_name: str,
         associated_slice: Optional[StreamSlice] = None,
-        file_reference: Optional[AirbyteRecordMessageFileReference] = None,
+        is_file_transfer_message: bool = False,
     ):
         self._data = data
         self._associated_slice = associated_slice
         self.stream_name = stream_name
-        self._file_reference = file_reference
+        self.is_file_transfer_message = is_file_transfer_message
 
     @property
     def data(self) -> Mapping[str, Any]:
@@ -39,14 +38,6 @@ class Record(Mapping[str, Any]):
     def associated_slice(self) -> Optional[StreamSlice]:
         return self._associated_slice
 
-    @property
-    def file_reference(self) -> AirbyteRecordMessageFileReference:
-        return self._file_reference
-
-    @file_reference.setter
-    def file_reference(self, value: AirbyteRecordMessageFileReference) -> None:
-        self._file_reference = value
-
     def __repr__(self) -> str:
         return repr(self._data)
 

airbyte_cdk/sources/utils/record_helper.py

@@ -9,10 +9,10 @@ from airbyte_cdk.models import (
     AirbyteLogMessage,
     AirbyteMessage,
     AirbyteRecordMessage,
-    AirbyteRecordMessageFileReference,
     AirbyteTraceMessage,
 )
 from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
 from airbyte_cdk.sources.streams.core import StreamData
 from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
 
@@ -22,7 +22,7 @@ def stream_data_to_airbyte_message(
     data_or_message: StreamData,
     transformer: TypeTransformer = TypeTransformer(TransformConfig.NoTransform),
     schema: Optional[Mapping[str, Any]] = None,
-    file_reference: Optional[AirbyteRecordMessageFileReference] = None,
+    is_file_transfer_message: bool = False,
 ) -> AirbyteMessage:
     if schema is None:
         schema = {}
@@ -36,12 +36,12 @@ def stream_data_to_airbyte_message(
             # taken unless configured. See
             # docs/connector-development/cdk-python/schemas.md for details.
             transformer.transform(data, schema)
-            message = AirbyteRecordMessage(
-                stream=stream_name,
-                data=data,
-                emitted_at=now_millis,
-                file_reference=file_reference,
-            )
+            if is_file_transfer_message:
+                message = AirbyteFileTransferRecordMessage(
+                    stream=stream_name, file=data, emitted_at=now_millis, data={}
+                )
+            else:
+                message = AirbyteRecordMessage(stream=stream_name, data=data, emitted_at=now_millis)
             return AirbyteMessage(type=MessageType.RECORD, record=message)
         case AirbyteTraceMessage():
             return AirbyteMessage(type=MessageType.TRACE, trace=data_or_message)

airbyte_cdk/test/mock_http/response_builder.py

@@ -198,14 +198,6 @@ def find_template(resource: str, execution_folder: str) -> Dict[str, Any]:
         return json.load(template_file)  # type: ignore  # we assume the dev correctly set up the resource file
 
 
-def find_binary_response(resource: str, execution_folder: str) -> bytes:
-    response_filepath = str(
-        get_unit_test_folder(execution_folder) / "resource" / "http" / "response" / f"{resource}"
-    )
-    with open(response_filepath, "rb") as response_file:
-        return response_file.read()  # type: ignore  # we assume the dev correctly set up the resource file
-
-
 def create_record_builder(
     response_template: Dict[str, Any],
     records_path: Union[FieldPath, NestedPath],

{airbyte_cdk-6.45.1.post47.dev14456468218.dist-info → airbyte_cdk-6.45.2.post3.dev14463482961.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.45.1.post47.dev14456468218
+Version: 6.45.2.post3.dev14463482961
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -22,7 +22,7 @@ Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: airbyte-protocol-models-dataclasses (>=0.15,<0.16)
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
 Requires-Dist: anyascii (>=0.3.2,<0.4.0)
 Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
 Requires-Dist: backoff

{airbyte_cdk-6.45.1.post47.dev14456468218.dist-info → airbyte_cdk-6.45.2.post3.dev14463482961.dist-info}/RECORD

@@ -7,13 +7,13 @@ airbyte_cdk/config_observation.py,sha256=7SSPxtN0nXPkm4euGNcTTr1iLbwUL01jy-24V1H
 airbyte_cdk/connector.py,sha256=bO23kdGRkl8XKFytOgrrWFc_VagteTHVEF6IsbizVkM,4224
 airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz52ftyNRnBw,1665
 airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=gwFpZHmvFrMB9NMlRuRfq0KEkbPaFucCB8k1DL20Wmw,5769
+airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=5ML_9Qw3TdD0NgHYhEOFNMvkmYCFLILbAX7kZFZx3gQ,5877
 airbyte_cdk/connector_builder/main.py,sha256=j1pP5N8RsnvQZ4iYxhLdLEHsJ5Ui7IVFBUi6wYMGBkM,3839
 airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
 airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
 airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=Iczn-_iczS2CaIAunWwyFcX0uLTra8Wh9JVfzm1Gfxo,26765
 airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=84BAEPIBHMq3WCfO14WNvh_q7OsjGgDt0q1FTu8eW-w,6918
-airbyte_cdk/connector_builder/test_reader/reader.py,sha256=GurMB4ITO_PntvhIHSJkXbhynLilI4DObY5A2axavXo,20667
+airbyte_cdk/connector_builder/test_reader/reader.py,sha256=wJtuYTZuc24-JlGF4UBFTJ2PChLjcQrvldqAWJM9Y9Y,20775
 airbyte_cdk/connector_builder/test_reader/types.py,sha256=hPZG3jO03kBaPyW94NI3JHRS1jxXGSNBcN1HFzOxo5Y,2528
 airbyte_cdk/destinations/__init__.py,sha256=FyDp28PT_YceJD5HDFhA-mrGfX9AONIyMQ4d68CHNxQ,213
 airbyte_cdk/destinations/destination.py,sha256=CIq-yb8C_0QvcKCtmStaHfiqn53GEfRAIGGCkJhKP1Q,5880
@@ -29,15 +29,16 @@ airbyte_cdk/destinations/vector_db_based/writer.py,sha256=nZ00xPiohElJmYktEZZIhr
 airbyte_cdk/entrypoint.py,sha256=NRJv5BNZRSUEVTmNBa9N7ih6fW5sg4DwL0nkB9kI99Y,18570
 airbyte_cdk/exception_handler.py,sha256=D_doVl3Dt60ASXlJsfviOCswxGyKF2q0RL6rif3fNks,2013
 airbyte_cdk/logger.py,sha256=1cURbvawbunCAV178q-XhTHcbAQZTSf07WhU7U9AXWU,3744
-airbyte_cdk/models/__init__.py,sha256=Et9wJWs5VOWynGbb-3aJRhsdAHAiLkNNLxdwqJAuqkw,2114
-airbyte_cdk/models/airbyte_protocol.py,sha256=oZdKsZ7yPjUt9hvxdWNpxCtgjSV2RWhf4R9Np03sqyY,3613
+airbyte_cdk/models/__init__.py,sha256=MOTiuML2wShBaMSIwikdjyye2uUWBjo4J1QFSbnoiM4,2075
+airbyte_cdk/models/airbyte_protocol.py,sha256=MCmLir67-hF12YM5OKzeGbWrlxr7ChG_OQSE1xG8EIU,3748
 airbyte_cdk/models/airbyte_protocol_serializers.py,sha256=s6SaFB2CMrG_7jTQGn_fhFbQ1FUxhCxf5kq2RWGHMVI,1749
+airbyte_cdk/models/file_transfer_record_message.py,sha256=J-E-43KOmUFdpsjeKlEfNnnZRSB-Gb5AGZjonR25Drc,323
 airbyte_cdk/models/well_known_types.py,sha256=EquepbisGPuCSrs_D7YVVnMR9-ShhUr21wnFz3COiJs,156
 airbyte_cdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/__init__.py,sha256=45J83QsFH3Wky3sVapZWg4C58R_i1thm61M06t2c1AQ,1156
 airbyte_cdk/sources/abstract_source.py,sha256=50vxEBRByiNhT4WJkiFvgM-C6PWqKSJgvuNC_aeg2cw,15547
 airbyte_cdk/sources/concurrent_source/__init__.py,sha256=3D_RJsxQfiLboSCDdNei1Iv-msRp3DXsas6E9kl7dXc,386
-airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py,sha256=P_GA5QayzehCf0ksUbEbGoNixBnauzsepv-0ICzhH4w,12691
+airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py,sha256=dbDBNcNNg2IZU5pZb3HfZeILU7X5_EhYGSbNqq3JD4I,12711
 airbyte_cdk/sources/concurrent_source/concurrent_source.py,sha256=P8B6EcLKaSstfAD9kDZsTJ0q8vRmdFrxLt-zOA5_By0,7737
 airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py,sha256=f9PIRPWn2tXu0-bxVeYHL2vYdqCzZ_kgpHg5_Ep-cfQ,6103
 airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py,sha256=z1t-rAZBsqVidv2fpUlPHE9JgyXsITuGk4AMu96mXSQ,696
@@ -66,11 +67,11 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=QeExVmpSYjr_CnghHu
 airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
 airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
 airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
-airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=nqqcSe2FeycTpYz3KsFomeOzRWxGYc2G5YfFNQrMKlI,28113
+airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=uhy0dRkA2jq89JdHZMTH5LdP200-Pm_LTaaWnCoUiaM,27687
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=OCZDuyHSoubti45_LHfFHyPyZDjeFBhY1CymG9_buBk,159887
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=pGFwleIC0sXFY-a6QphWyjY9QjUfHasgChPAMZFWjU8,158152
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
@@ -88,7 +89,7 @@ airbyte_cdk/sources/declarative/extractors/dpath_extractor.py,sha256=wR4Ol4MG2lt
 airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8VwkjS0xD_u350Km3SiYP7hpOOgiLg5o,1169
 airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
 airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
-airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=Xg4e0exJ5Tq8I346uD2-HjpCsAUFLgPPcNKa0UoHjV8,7178
+airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=HCqx7IyENM_aRF4it2zJN26_vDu6WeP8XgCxQWHUvcY,6934
 airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=WJyA2OYIEgFpVP5Y3o0tIj69AV6IKkn9B16MeXaEItI,6513
 airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
 airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
@@ -113,13 +114,13 @@ airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=EYPTtYTxNm_u-8OacYF6d4ce0PdV6jzYnxTmpjZOu_g,113491
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=EfhCFVz_UHl7_cbRlf5hvFO1J5osxAZjISeqgbXE2t8,112007
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=4C15MKV-zOrMVQAm4FyohDsrJUBCSpMv5tZw0SK3aeI,9685
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=2mB379Yg7oD11j6HkuGq2OyOgmE2Sc4itPWo9XhI_hs,160128
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Qim2qR64SKWbxD-Au0KAB3fLf6G2YfppUv3Q0w4QYtE,158619
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -178,9 +179,8 @@ airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=d
 airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=nQepwG_RfW53sgwvK5dLPqfCx0VjsQ83nYoPjBMAaLM,527
 airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=6oZtnCHm9NdDvjTSrVwPQOXGSdETSIR7eWH2vFjM7jI,4855
-airbyte_cdk/sources/declarative/retrievers/file_uploader.py,sha256=Om8vaDHix_1EXxEDVpRP57R94KmBmzJJLWtgxDJ5P8E,3363
 airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
-airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=Ixs_byM-DMfYiYB-iOa97yH4WGAvxgTdTYeAsskHdYM,31134
+airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=cI3UEWSIuGNzzIlf8I_7Vf_3fX_tQwIwPrnmrY7MEh4,31146
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
 airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=UnbzlExmwoQiVV8zDg4lhAEaqA_0pRfwbMRe8yqOuWk,1834
 airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=J8Q_iJYhcSQLWyt0bTZCbDAGpxt9G8FCc6Q9jtGsNzw,10703
@@ -190,7 +190,7 @@ airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLn
 airbyte_cdk/sources/declarative/spec/__init__.py,sha256=H0UwoRhgucbKBIzg85AXrifybVmfpwWpPdy22vZKVuo,141
 airbyte_cdk/sources/declarative/spec/spec.py,sha256=ODSNUgkDOhnLQnwLjgSaME6R3kNeywjROvbNrWEnsgU,1876
 airbyte_cdk/sources/declarative/stream_slicers/__init__.py,sha256=sI9vhc95RwJYOnA0VKjcbtKgFcmAbWjhdWBXFbAijOs,176
-airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=cjKGm4r438dd1GxrFHJ4aYrdzG2bkncnwaWxAwlXR3M,3585
+airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=RW1Q44ml-VWeMl4lNcV6EfyzrzCZkjj-hd0Omx_n_n4,3405
 airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
 airbyte_cdk/sources/declarative/transformations/__init__.py,sha256=CPJ8TlMpiUmvG3624VYu_NfTzxwKcfBjM2Q2wJ7fkSA,919
 airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=Eg1jQtRObgzxbtySTQs5uEZIjEklsoHFxYSPf78x6Ng,5420
@@ -224,26 +224,25 @@ airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py,sha2
 airbyte_cdk/sources/file_based/exceptions.py,sha256=WP0qkG6fpWoBpOyyicgp5YNE393VWyegq5qSy0v4QtM,7362
 airbyte_cdk/sources/file_based/file_based_source.py,sha256=Xg8OYWnGc-OcVBglvS08uwAWGWHBhEqsBnyODIkOK-4,20051
 airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py,sha256=4e7FXqQ9hueacexC0SyrZyjF8oREYHza8pKF9CgKbD8,5050
-airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=rwz8AhEIqYB9gBF7uW9eR--eUiHOntzuwLH8jFHNacE,7854
-airbyte_cdk/sources/file_based/file_record_data.py,sha256=aIbUHHjZOF_OIS22ePr0Vu0MQ6rmLmSC1D4kDdGmVKw,415
+airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=0cmppYO3pZlFiJrs5oorF4JXv4ErhOeEMrdLG7P-Gdk,6742
 airbyte_cdk/sources/file_based/file_types/__init__.py,sha256=blCLn0-2LC-ZdgcNyDEhqM2RiUvEjEBh-G4-t32ZtuM,1268
 airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=USEYqiICXBWpDV443VtNOCmUA-GINzY_Zah74_5w3qQ,10860
 airbyte_cdk/sources/file_based/file_types/csv_parser.py,sha256=QlCXB-ry3np67Q_VerQEPoWDOTcPTB6Go4ydZxY9ae4,20445
 airbyte_cdk/sources/file_based/file_types/excel_parser.py,sha256=BeplCq0hmojELU6bZCvvpRLpQ9us81TqbGYwrhd3INo,7188
-airbyte_cdk/sources/file_based/file_types/file_transfer.py,sha256=5l2Jo6bp6neDmgM427PrZMZeqU0hCIZVWnzUZ_7BT10,1100
+airbyte_cdk/sources/file_based/file_types/file_transfer.py,sha256=HyGRihJxcb_lEsffKhfF3eylLBDy51_PXSwGUFEJ5bA,1265
 airbyte_cdk/sources/file_based/file_types/file_type_parser.py,sha256=JgpH21PrbRqwK92BJklZWvh2TndA6xZ-eP1LPMo44oQ,2832
 airbyte_cdk/sources/file_based/file_types/jsonl_parser.py,sha256=GwyNyxmST4RX-XpXy7xVH0D-znYWWBmGv_pVAu95oHQ,5886
 airbyte_cdk/sources/file_based/file_types/parquet_parser.py,sha256=XenFg5sJ-UBnIkSmsiNJRou11NO0zZXx-RXgPHMT2NA,10487
 airbyte_cdk/sources/file_based/file_types/unstructured_parser.py,sha256=2TYOQl62FQPCa8otLbkDIk_j01EP3oWaKSfXGhCjCHg,19492
 airbyte_cdk/sources/file_based/remote_file.py,sha256=yqRz93vPe8PBXLIMJ5W5u2JRlZRhg6sBrAjn3pPjJ8A,315
-airbyte_cdk/sources/file_based/schema_helpers.py,sha256=1zRxdyirn-1kx2Zvh1TOZ_AzFkwrNxqwIZFmQ5V-2s4,9859
+airbyte_cdk/sources/file_based/schema_helpers.py,sha256=Cf8FH1bDFP0qCDDfEYir_WjP4exXUnikz8hZ40y1Ek0,9601
 airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py,sha256=FkByIyEy56x2_awYnxGPqGaOp7zAzpAoRkPZHKySI9M,536
 airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py,sha256=kjvX7nOmUALYd7HuZHilUzgJPZ-MnZ08mtvuBnt2tQ0,618
 airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py,sha256=vjTlmYT_nqzY3DbT5xem7X-bwgA9RyXHoKFqiMO2URk,1728
 airbyte_cdk/sources/file_based/stream/__init__.py,sha256=q_zmeOHHg0JK5j1YNSOIsyXGz-wlTl_0E8z5GKVAcVM,543
 airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=9pQh3BHYcxm8CRC8XawfmBxL8O9HggpWwCCbX_ncINE,7509
 airbyte_cdk/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=1AIuAOHa_M6zN9l0eAWBHwhKl4fdP4-KlUMOMzTv11U,13525
+airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=EwWuoCsQRNaoizxbb2-BYVwRYOxr57exw3q3M6zVv1E,13931
 airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py,sha256=Rx7TwjH8B7e0eee83Tlqxv1bWn-BVXOmlUAH7auM1uM,344
 airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py,sha256=5dYZMLBEbvCyrCT89lCYdm2FdrLPLuxjdpQSVGP5o0w,1856
 airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py,sha256=gRTL-9I3ejjQOpLKd6ixe9rB3kGlubCdhUt9ri6AdAI,14880
@@ -251,9 +250,9 @@ airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_c
 airbyte_cdk/sources/file_based/stream/cursor/__init__.py,sha256=MhFB5hOo8sjwvCh8gangaymdg3EJWYt_72brFOZt068,191
 airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py,sha256=om-x3gZFPgWDpi15S9RxZmR36VHnk8sytgN6LlBQhAw,1934
 airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py,sha256=VGV7xLyBribuBMVrXtO1xqkWJD86bl7yhXtjnwLMohM,7051
-airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=gzSN4cLywwzo_U1OdFS3Of_-4DRkUcX_j7Mv30MrxQs,17154
+airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=jyJLu2BUCYWKqrqD0ZUFxnrD0qybny7KbzKznxjIIpM,18199
 airbyte_cdk/sources/file_based/stream/identities_stream.py,sha256=FZH83Geoy3K3nwUk2VVNJERFcXUTnl-4XljjucUM23s,1893
-airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py,sha256=6KxqdD3-VvwxDTk7TtZ0M32fga4CI3qZ9IKdAkySpx0,3844
+airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py,sha256=ke82qgm7snOlQTDx94Lqsc0cDkHWi3OJDTrPxffpFqc,3914
 airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
 airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
 airbyte_cdk/sources/http_logger.py,sha256=H93kPAujHhPmXNX0JSFG3D-SL6yEFA5PtKot9Hu3TYA,1690
@@ -279,7 +278,7 @@ airbyte_cdk/sources/streams/concurrent/availability_strategy.py,sha256=4La5v2Uff
 airbyte_cdk/sources/streams/concurrent/clamping.py,sha256=i26GVyui2ScEXSP-IP_61K2HaTp1-6lTlYHsZVYpuZA,3240
 airbyte_cdk/sources/streams/concurrent/cursor.py,sha256=LFXbKBEMtNSVz_kZs9qydS9fPvzTU5wdgXRagRRJeHo,21388
 airbyte_cdk/sources/streams/concurrent/cursor_types.py,sha256=ZyWLPpeLX1qXcP5MwS-wxK11IBMsnVPCw9zx8gA2_Ro,843
-airbyte_cdk/sources/streams/concurrent/default_stream.py,sha256=3SBjFa1z955pSE_2qt1C7mAky-RKjOZeQDePbZkWYYs,3371
+airbyte_cdk/sources/streams/concurrent/default_stream.py,sha256=K3rLMpYhS7nnmvwQ52lqBy7DQdFMJpvvT7sgBg_ckA8,3207
 airbyte_cdk/sources/streams/concurrent/exceptions.py,sha256=JOZ446MCLpmF26r9KfS6OO_6rGjcjgJNZdcw6jccjEI,468
 airbyte_cdk/sources/streams/concurrent/helpers.py,sha256=S6AW8TgIASCZ2UuUcQLE8OzgYUHWt2-KPOvNPwnQf-Q,1596
 airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py,sha256=2t64b_z9cEPmlHZnjSiMTO8PEtEdiAJDG0JcYOtUqAE,3363
@@ -316,11 +315,10 @@ airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=C2j2uVfi9d
 airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
 airbyte_cdk/sources/streams/permissions/identities_stream.py,sha256=9O9k6k18Xm3Zsiw_vnI_jsHXfMCQiek6V-jMkJJLxn8,2621
 airbyte_cdk/sources/streams/utils/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-airbyte_cdk/sources/types.py,sha256=1oerHQpidFrIluUhWsM3-4Xst4wwUDqzkgtgZY0jjZU,5485
+airbyte_cdk/sources/types.py,sha256=x5Rkfpottg46qgr1-g_O4kN6v0Fd0sWHttdYsftyo7w,5148
 airbyte_cdk/sources/utils/__init__.py,sha256=TTN6VUxVy6Is8BhYQZR5pxJGQh8yH4duXh4O1TiMiEY,118
 airbyte_cdk/sources/utils/casing.py,sha256=QC-gV1O4e8DR4-bhdXieUPKm_JamzslVyfABLYYRSXA,256
-airbyte_cdk/sources/utils/files_directory.py,sha256=z8Dmr-wkL1sAqdwCST4MBUFAyMHPD2cJIzVdAuCynp8,391
-airbyte_cdk/sources/utils/record_helper.py,sha256=7wL-pDYrBpcmZHa8ORtiSOqBZJEZI5hdl2dA1RYiatk,2029
+airbyte_cdk/sources/utils/record_helper.py,sha256=jeB0mucudzna7Zvj-pCBbwFrbLJ36SlAWZTh5O4Fb9Y,2168
 airbyte_cdk/sources/utils/schema_helpers.py,sha256=bR3I70-e11S6B8r6VK-pthQXtcYrXojgXFvuK7lRrpg,8545
 airbyte_cdk/sources/utils/slice_logger.py,sha256=qWWeFLAvigFz0b4O1_O3QDM1cy8PqZAMMgVPR2hEeb8,1778
 airbyte_cdk/sources/utils/transform.py,sha256=0LOvIJg1vmg_70AiAVe-YHMr-LHrqEuxg9cm1BnYPDM,11725
@@ -344,7 +342,7 @@ airbyte_cdk/test/mock_http/matcher.py,sha256=4Qj8UnJKZIs-eodshryce3SN1Ayc8GZpBET
 airbyte_cdk/test/mock_http/mocker.py,sha256=XgsjMtVoeMpRELPyALgrkHFauH9H5irxrz1Kcxh2yFY,8013
 airbyte_cdk/test/mock_http/request.py,sha256=tdB8cqk2vLgCDTOKffBKsM06llYs4ZecgtH6DKyx6yY,4112
 airbyte_cdk/test/mock_http/response.py,sha256=s4-cQQqTtmeej0pQDWqmG0vUWpHS-93lIWMpW3zSVyU,662
-airbyte_cdk/test/mock_http/response_builder.py,sha256=F-v7ebftqGj7YVIMLKdodmU9U8Dq8aIyllWGo2NGwHc,8331
+airbyte_cdk/test/mock_http/response_builder.py,sha256=debPx_lRYBaQVSwCoKLa0F8KFk3h0qG7bWxFBATa0cc,7958
 airbyte_cdk/test/state_builder.py,sha256=kLPql9lNzUJaBg5YYRLJlY_Hy5JLHJDVyKPMZMoYM44,946
 airbyte_cdk/test/utils/__init__.py,sha256=Hu-1XT2KDoYjDF7-_ziDwv5bY3PueGjANOCbzeOegDg,57
 airbyte_cdk/test/utils/data.py,sha256=CkCR1_-rujWNmPXFR1IXTMwx1rAl06wAyIKWpDcN02w,820
@@ -368,9 +366,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.45.1.post47.dev14456468218.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.45.1.post47.dev14456468218.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.45.1.post47.dev14456468218.dist-info/METADATA,sha256=E2z0nkONKG5sYIjIZdGW6ahxFjfLAUH2YYx66TWzlGA,6093
-airbyte_cdk-6.45.1.post47.dev14456468218.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.45.1.post47.dev14456468218.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.45.1.post47.dev14456468218.dist-info/RECORD,,
+airbyte_cdk-6.45.2.post3.dev14463482961.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.45.2.post3.dev14463482961.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.45.2.post3.dev14463482961.dist-info/METADATA,sha256=1zWGqhfb96Ud4180ayG4dERFE_OGwnWsUQqs8eUO38k,6092
+airbyte_cdk-6.45.2.post3.dev14463482961.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.45.2.post3.dev14463482961.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.45.2.post3.dev14463482961.dist-info/RECORD,,

airbyte_cdk/sources/declarative/retrievers/file_uploader.py

@@ -1,89 +0,0 @@
-#
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-#
-
-import json
-import logging
-import uuid
-from dataclasses import InitVar, dataclass, field
-from pathlib import Path
-from typing import Any, Mapping, Optional, Union
-
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
-from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
-from airbyte_cdk.sources.declarative.interpolation.interpolated_string import (
-    InterpolatedString,
-)
-from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import (
-    SafeResponse,
-)
-from airbyte_cdk.sources.declarative.requesters import Requester
-from airbyte_cdk.sources.declarative.types import Record, StreamSlice
-from airbyte_cdk.sources.types import Config
-from airbyte_cdk.sources.utils.files_directory import get_files_directory
-
-logger = logging.getLogger("airbyte")
-
-
-@dataclass
-class FileUploader:
-    requester: Requester
-    download_target_extractor: RecordExtractor
-    config: Config
-    parameters: InitVar[Mapping[str, Any]]
-
-    filename_extractor: Optional[Union[InterpolatedString, str]] = None
-    content_extractor: Optional[RecordExtractor] = None
-
-    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        if self.filename_extractor:
-            self.filename_extractor = InterpolatedString.create(
-                self.filename_extractor,
-                parameters=parameters,
-            )
-
-    def upload(self, record: Record) -> None:
-        mocked_response = SafeResponse()
-        mocked_response.content = json.dumps(record.data).encode()
-        download_target = list(self.download_target_extractor.extract_records(mocked_response))[0]
-        if not isinstance(download_target, str):
-            raise ValueError(
-                f"download_target is expected to be a str but was {type(download_target)}: {download_target}"
-            )
-
-        response = self.requester.send_request(
-            stream_slice=StreamSlice(
-                partition={}, cursor_slice={}, extra_fields={"download_target": download_target}
-            ),
-        )
-
-        if self.content_extractor:
-            raise NotImplementedError("TODO")
-        else:
-            files_directory = Path(get_files_directory())
-
-            file_name = (
-                self.filename_extractor.eval(self.config, record=record)
-                if self.filename_extractor
-                else str(uuid.uuid4())
-            )
-            file_name = file_name.lstrip("/")
-            file_relative_path = Path(record.stream_name) / Path(file_name)
-
-            full_path = files_directory / file_relative_path
-            full_path.parent.mkdir(parents=True, exist_ok=True)
-
-            with open(str(full_path), "wb") as f:
-                f.write(response.content)
-            file_size_bytes = full_path.stat().st_size
-
-            logger.info("File uploaded successfully")
-            logger.info(f"File url: {str(full_path)}")
-            logger.info(f"File size: {file_size_bytes / 1024} KB")
-            logger.info(f"File relative path: {str(file_relative_path)}")
-
-            record.file_reference = AirbyteRecordMessageFileReference(
-                staging_file_url=str(full_path),
-                source_file_relative_path=str(file_relative_path),
-                file_size_bytes=file_size_bytes,
-            )

airbyte_cdk/sources/file_based/file_record_data.py

@@ -1,22 +0,0 @@
-#
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-#
-
-from datetime import datetime
-from typing import Optional
-
-from pydantic.v1 import BaseModel
-
-
-class FileRecordData(BaseModel):
-    """
-    A record in a file-based stream.
-    """
-
-    folder: str
-    filename: str
-    bytes: int
-    source_uri: str
-    id: Optional[str] = None
-    updated_at: Optional[str] = None
-    mime_type: Optional[str] = None

airbyte_cdk/sources/utils/files_directory.py

@@ -1,15 +0,0 @@
-#
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-#
-import os
-
-AIRBYTE_STAGING_DIRECTORY = os.getenv("AIRBYTE_STAGING_DIRECTORY", "/staging/files")
-DEFAULT_LOCAL_DIRECTORY = "/tmp/airbyte-file-transfer"
-
-
-def get_files_directory() -> str:
-    return (
-        AIRBYTE_STAGING_DIRECTORY
-        if os.path.exists(AIRBYTE_STAGING_DIRECTORY)
-        else DEFAULT_LOCAL_DIRECTORY
-    )