airbyte-cdk 6.47.0.dev0__py3-none-any.whl → 6.47.2__py3-none-any.whl

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -305,6 +305,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                                 stream_name=declarative_stream.name,
                                 stream_namespace=declarative_stream.namespace,
                                 config=config or {},
+                                stream_state_migrations=declarative_stream.state_migrations,
                             )
                         partition_generator = StreamSlicerPartitionGenerator(
                             partition_factory=DeclarativePartitionFactory(

airbyte_cdk/sources/declarative/extractors/record_selector.py

@@ -15,7 +15,7 @@ from airbyte_cdk.sources.declarative.extractors.type_transformer import (
 )
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.models import SchemaNormalization
-from airbyte_cdk.sources.declarative.retrievers.file_uploader import FileUploader
+from airbyte_cdk.sources.declarative.retrievers.file_uploader import DefaultFileUploader
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
 from airbyte_cdk.sources.utils.transform import TypeTransformer
@@ -43,7 +43,7 @@ class RecordSelector(HttpSelector):
     record_filter: Optional[RecordFilter] = None
     transformations: List[RecordTransformation] = field(default_factory=lambda: [])
     transform_before_filtering: bool = False
-    file_uploader: Optional[FileUploader] = None
+    file_uploader: Optional[DefaultFileUploader] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -481,7 +481,13 @@ from airbyte_cdk.sources.declarative.retrievers import (
     SimpleRetriever,
     SimpleRetrieverTestReadDecorator,
 )
-from airbyte_cdk.sources.declarative.retrievers.file_uploader import FileUploader
+from airbyte_cdk.sources.declarative.retrievers.file_uploader import (
+    ConnectorBuilderFileUploader,
+    DefaultFileUploader,
+    FileUploader,
+    LocalFileSystemFileWriter,
+    NoopFileWriter,
+)
 from airbyte_cdk.sources.declarative.schema import (
     ComplexFieldType,
     DefaultSchemaLoader,
@@ -2815,7 +2821,7 @@ class ModelToComponentFactory:
         transformations: List[RecordTransformation] | None = None,
         decoder: Decoder | None = None,
         client_side_incremental_sync: Dict[str, Any] | None = None,
-        file_uploader: Optional[FileUploader] = None,
+        file_uploader: Optional[DefaultFileUploader] = None,
         **kwargs: Any,
     ) -> RecordSelector:
         extractor = self._create_component_from_model(
@@ -2919,7 +2925,7 @@ class ModelToComponentFactory:
         stop_condition_on_cursor: bool = False,
         client_side_incremental_sync: Optional[Dict[str, Any]] = None,
         transformations: List[RecordTransformation],
-        file_uploader: Optional[FileUploader] = None,
+        file_uploader: Optional[DefaultFileUploader] = None,
         incremental_sync: Optional[
             Union[
                 IncrementingCountCursorModel, DatetimeBasedCursorModel, CustomIncrementalSyncModel
@@ -3606,14 +3612,24 @@ class ModelToComponentFactory:
             name=name,
             **kwargs,
         )
-        return FileUploader(
+        emit_connector_builder_messages = self._emit_connector_builder_messages
+        file_uploader = DefaultFileUploader(
             requester=requester,
             download_target_extractor=download_target_extractor,
             config=config,
+            file_writer=NoopFileWriter()
+            if emit_connector_builder_messages
+            else LocalFileSystemFileWriter(),
             parameters=model.parameters or {},
             filename_extractor=model.filename_extractor if model.filename_extractor else None,
         )
 
+        return (
+            ConnectorBuilderFileUploader(file_uploader)
+            if emit_connector_builder_messages
+            else file_uploader
+        )
+
     def create_moving_window_call_rate_policy(
         self, model: MovingWindowCallRatePolicyModel, config: Config, **kwargs: Any
     ) -> MovingWindowCallRatePolicy:

airbyte_cdk/sources/declarative/retrievers/file_uploader/__init__.py

@@ -0,0 +1,15 @@
+from .connector_builder_file_uploader import ConnectorBuilderFileUploader
+from .default_file_uploader import DefaultFileUploader
+from .file_uploader import FileUploader
+from .file_writer import FileWriter
+from .local_file_system_file_writer import LocalFileSystemFileWriter
+from .noop_file_writer import NoopFileWriter
+
+__all__ = [
+    "DefaultFileUploader",
+    "LocalFileSystemFileWriter",
+    "NoopFileWriter",
+    "ConnectorBuilderFileUploader",
+    "FileUploader",
+    "FileWriter",
+]

airbyte_cdk/sources/declarative/retrievers/file_uploader/connector_builder_file_uploader.py

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import dataclass
+
+from airbyte_cdk.sources.declarative.types import Record
+
+from .default_file_uploader import DefaultFileUploader
+from .file_uploader import FileUploader
+
+
+@dataclass
+class ConnectorBuilderFileUploader(FileUploader):
+    """
+    Connector builder file uploader
+    Acts as a decorator or wrapper around a FileUploader instance, copying the attributes from record.file_reference into the record.data.
+    """
+
+    file_uploader: DefaultFileUploader
+
+    def upload(self, record: Record) -> None:
+        self.file_uploader.upload(record=record)
+        for file_reference_key, file_reference_value in record.file_reference.__dict__.items():
+            if not file_reference_key.startswith("_"):
+                record.data[file_reference_key] = file_reference_value  # type: ignore

airbyte_cdk/sources/declarative/retrievers/{file_uploader.py → file_uploader/default_file_uploader.py}

@@ -22,18 +22,27 @@ from airbyte_cdk.sources.declarative.types import Record, StreamSlice
 from airbyte_cdk.sources.types import Config
 from airbyte_cdk.sources.utils.files_directory import get_files_directory
 
+from .file_uploader import FileUploader
+from .file_writer import FileWriter
+
 logger = logging.getLogger("airbyte")
 
 
 @dataclass
-class FileUploader:
+class DefaultFileUploader(FileUploader):
+    """
+    File uploader class
+    Handles the upload logic: fetching the download target, making the request via its requester, determining the file path, and calling self.file_writer.write()
+    Different types of file_writer:BaseFileWriter can be injected to handle different file writing strategies.
+    """
+
     requester: Requester
     download_target_extractor: RecordExtractor
     config: Config
+    file_writer: FileWriter
     parameters: InitVar[Mapping[str, Any]]
 
     filename_extractor: Optional[Union[InterpolatedString, str]] = None
-    content_extractor: Optional[RecordExtractor] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         if self.filename_extractor:
@@ -61,33 +70,28 @@ class FileUploader:
             ),
         )
 
-        if self.content_extractor:
-            raise NotImplementedError("TODO")
-        else:
-            files_directory = Path(get_files_directory())
+        files_directory = Path(get_files_directory())
 
-            file_name = (
-                self.filename_extractor.eval(self.config, record=record)
-                if self.filename_extractor
-                else str(uuid.uuid4())
-            )
-            file_name = file_name.lstrip("/")
-            file_relative_path = Path(record.stream_name) / Path(file_name)
+        file_name = (
+            self.filename_extractor.eval(self.config, record=record)
+            if self.filename_extractor
+            else str(uuid.uuid4())
+        )
+        file_name = file_name.lstrip("/")
+        file_relative_path = Path(record.stream_name) / Path(file_name)
 
-            full_path = files_directory / file_relative_path
-            full_path.parent.mkdir(parents=True, exist_ok=True)
+        full_path = files_directory / file_relative_path
+        full_path.parent.mkdir(parents=True, exist_ok=True)
 
-            with open(str(full_path), "wb") as f:
-                f.write(response.content)
-            file_size_bytes = full_path.stat().st_size
+        file_size_bytes = self.file_writer.write(full_path, content=response.content)
 
-            logger.info("File uploaded successfully")
-            logger.info(f"File url: {str(full_path)}")
-            logger.info(f"File size: {file_size_bytes / 1024} KB")
-            logger.info(f"File relative path: {str(file_relative_path)}")
+        logger.info("File uploaded successfully")
+        logger.info(f"File url: {str(full_path)}")
+        logger.info(f"File size: {file_size_bytes / 1024} KB")
+        logger.info(f"File relative path: {str(file_relative_path)}")
 
-            record.file_reference = AirbyteRecordMessageFileReference(
-                staging_file_url=str(full_path),
-                source_file_relative_path=str(file_relative_path),
-                file_size_bytes=file_size_bytes,
-            )
+        record.file_reference = AirbyteRecordMessageFileReference(
+            staging_file_url=str(full_path),
+            source_file_relative_path=str(file_relative_path),
+            file_size_bytes=file_size_bytes,
+        )

airbyte_cdk/sources/declarative/retrievers/file_uploader/file_uploader.py

@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+from airbyte_cdk.sources.declarative.types import Record
+
+
+@dataclass
+class FileUploader(ABC):
+    """
+    Base class for file uploader
+    """
+
+    @abstractmethod
+    def upload(self, record: Record) -> None:
+        """
+        Uploads the file to the specified location
+        """
+        ...

airbyte_cdk/sources/declarative/retrievers/file_uploader/file_writer.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class FileWriter(ABC):
+    """
+    Base File writer class
+    """
+
+    @abstractmethod
+    def write(self, file_path: Path, content: bytes) -> int:
+        """
+        Writes the file to the specified location
+        """
+        ...

airbyte_cdk/sources/declarative/retrievers/file_uploader/local_file_system_file_writer.py

@@ -0,0 +1,18 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from pathlib import Path
+
+from .file_writer import FileWriter
+
+
+class LocalFileSystemFileWriter(FileWriter):
+    def write(self, file_path: Path, content: bytes) -> int:
+        """
+        Writes the file to the specified location
+        """
+        with open(str(file_path), "wb") as f:
+            f.write(content)
+
+        return file_path.stat().st_size

airbyte_cdk/sources/declarative/retrievers/file_uploader/noop_file_writer.py

@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from pathlib import Path
+
+from .file_writer import FileWriter
+
+
+class NoopFileWriter(FileWriter):
+    NOOP_FILE_SIZE = -1
+
+    def write(self, file_path: Path, content: bytes) -> int:
+        """
+        Noop file writer
+        """
+        return self.NOOP_FILE_SIZE

{airbyte_cdk-6.47.0.dev0.dist-info → airbyte_cdk-6.47.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.47.0.dev0
+Version: 6.47.2
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.47.0.dev0.dist-info → airbyte_cdk-6.47.2.dist-info}/RECORD

@@ -74,7 +74,7 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=QeExVmpSYjr_CnghHu
 airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
 airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
 airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
-airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=mB0wBAuAIvIJwRsMLoLGUoSuMimcY1tgyUjPiTHHHZE,28269
+airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=GoZJ8Oxb3IQh6iJao6dw7NNiWVyWTrlUlWVKMWt-V0k,28362
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
@@ -96,7 +96,7 @@ airbyte_cdk/sources/declarative/extractors/dpath_extractor.py,sha256=wR4Ol4MG2lt
 airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8VwkjS0xD_u350Km3SiYP7hpOOgiLg5o,1169
 airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
 airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
-airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=Xg4e0exJ5Tq8I346uD2-HjpCsAUFLgPPcNKa0UoHjV8,7178
+airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=vCpwX1PVRFPYKMzm0DHHP3YEZ0Gmd3bBzggOsRha038,7192
 airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=WJyA2OYIEgFpVP5Y3o0tIj69AV6IKkn9B16MeXaEItI,6513
 airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
 airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
@@ -128,7 +128,7 @@ airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9R
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=RUyFZS0zslLb7UfQrvqMC--k5CVLNSp7zHw6kbosvKE,9688
 airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=3sHQPrJrgIsk_D17g3vf61JfD4MEJDbniR1JqOyMb64,161761
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=JX5YAUoTGIjTxIGJoIbRoOvk2piHBNnPQDseSOQzoR0,162286
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -187,7 +187,13 @@ airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=d
 airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=nQepwG_RfW53sgwvK5dLPqfCx0VjsQ83nYoPjBMAaLM,527
 airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=6oZtnCHm9NdDvjTSrVwPQOXGSdETSIR7eWH2vFjM7jI,4855
-airbyte_cdk/sources/declarative/retrievers/file_uploader.py,sha256=J-OjleRStGAGaoPylttyFQwVGHvtFTqdfnS0JYcYUtg,3537
+airbyte_cdk/sources/declarative/retrievers/file_uploader/__init__.py,sha256=ovs0oZGPEEw-V4asqr0D1Xty2u7tR8ADhRuHNgXGuWQ,490
+airbyte_cdk/sources/declarative/retrievers/file_uploader/connector_builder_file_uploader.py,sha256=wHGnluTJ6HVbCkSaNNQQN2t9ZzhaFGoo_pzXTz54gBY,878
+airbyte_cdk/sources/declarative/retrievers/file_uploader/default_file_uploader.py,sha256=7AFJxJknE7whofFBxoBp-EUrpu2QyZh1dwEq1jb3D-I,3689
+airbyte_cdk/sources/declarative/retrievers/file_uploader/file_uploader.py,sha256=bWCWQzPokZ9rWmmi3I6RfaG53FhSN5yj3JouRhtnz7E,431
+airbyte_cdk/sources/declarative/retrievers/file_uploader/file_writer.py,sha256=V8gAFjQXkhX5mwj1NafdcUrMfMBNF1hi0mrdXIl5qEc,359
+airbyte_cdk/sources/declarative/retrievers/file_uploader/local_file_system_file_writer.py,sha256=jLpdonre1UHfbjGSD5AK_T0codLABJByTvbqepDZtEQ,422
+airbyte_cdk/sources/declarative/retrievers/file_uploader/noop_file_writer.py,sha256=1yfimzxm09d2j605cu_HhiYVDNVL1rUMi3vs_jYlIyY,330
 airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
 airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=O7qpM71L1_ATIbEKa8y658jdiSJSPw0KmuGKgnaruQU,31008
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
@@ -390,9 +396,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.47.0.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.47.0.dev0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.47.0.dev0.dist-info/METADATA,sha256=w15gBn52dA5wRUmZ_w1t2ViE2CDJimBiS4PWfjnK2vg,6328
-airbyte_cdk-6.47.0.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.47.0.dev0.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.47.0.dev0.dist-info/RECORD,,
+airbyte_cdk-6.47.2.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.47.2.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.47.2.dist-info/METADATA,sha256=hKm118q6Iw_E3CKxiCZjXAPauBtssPkRvsmRVzeomvY,6323
+airbyte_cdk-6.47.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.47.2.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.47.2.dist-info/RECORD,,