airbyte-cdk 6.13.0__py3-none-any.whl → 6.13.1.dev4100__py3-none-any.whl

airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py

@@ -83,9 +83,7 @@ class OffsetIncrement(PaginationStrategy):
             return self._offset
 
     def reset(self, reset_value: Optional[Any] = 0) -> None:
-        if reset_value is None:
-            self._offset = 0
-        elif not isinstance(reset_value, int):
+        if not isinstance(reset_value, int):
             raise ValueError(
                 f"Reset value {reset_value} for OffsetIncrement pagination strategy was not an integer"
             )

airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py

@@ -53,10 +53,7 @@ class StopConditionPaginationStrategyDecorator(PaginationStrategy):
         return self._delegate.next_page_token(response, last_page_size, last_record)
 
     def reset(self, reset_value: Optional[Any] = None) -> None:
-        if reset_value:
-            self._delegate.reset(reset_value)
-        else:
-            self._delegate.reset()
+        self._delegate.reset(reset_value)
 
     def get_page_size(self) -> Optional[int]:
         return self._delegate.get_page_size()

airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py

@@ -14,6 +14,13 @@ from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileB
 from airbyte_cdk.sources.utils import schema_helpers
 
 
+class DeliveryOptions(BaseModel):
+    preserve_subdirectories_directories: bool = Field(
+        True,
+        description="Flag indicating we should preserve subdirectories directories",
+    )
+
+
 class DeliverRecords(BaseModel):
     class Config(OneOfOptionConfig):
         title = "Replicate Records"
@@ -30,6 +37,11 @@ class DeliverRawFiles(BaseModel):
         discriminator = "delivery_type"
 
     delivery_type: Literal["use_file_transfer"] = Field("use_file_transfer", const=True)
+    delivery_options: Optional[DeliveryOptions] = Field(
+        title="Delivery Options",
+        type="object",
+        order=2,
+    )
 
 
 class AbstractFileBasedSpec(BaseModel):
@@ -65,6 +77,12 @@ class AbstractFileBasedSpec(BaseModel):
         airbyte_hidden=True,
     )
 
+    delivery_options: Optional[DeliveryOptions] = Field(
+        title="Delivery Options",
+        type="object",
+        order=8,
+    )
+
     @classmethod
     @abstractmethod
     def documentation_url(cls) -> AnyUrl:

airbyte_cdk/sources/file_based/exceptions.py

@@ -111,6 +111,10 @@ class ErrorListingFiles(BaseFileBasedSourceError):
     pass
 
 
+class DuplicatedFilesError(BaseFileBasedSourceError):
+    pass
+
+
 class CustomFileBasedException(AirbyteTracedException):
     """
     A specialized exception for file-based connectors.
@@ -123,3 +127,25 @@ class CustomFileBasedException(AirbyteTracedException):
 
 class FileSizeLimitError(CustomFileBasedException):
     pass
+
+
+def format_duplicate_files_error_message(
+    stream_name: str, duplicated_files_names: List[dict[str, List[str]]]
+) -> str:
+    duplicated_files_messages = []
+    for duplicated_file in duplicated_files_names:
+        for duplicated_file_name, file_paths in duplicated_file.items():
+            file_duplicated_message = (
+                f"{len(file_paths)} duplicates found for file name {duplicated_file_name}:\n\n"
+                + "".join(f"\n - {file_paths}")
+            )
+            duplicated_files_messages.append(file_duplicated_message)
+
+    error_message = (
+        f"ERROR: Duplicate filenames found for stream {stream_name}. "
+        "Duplicate file names are not allowed if the Preserve Subdirectories in File Paths option is disabled. "
+        "Please remove or rename the duplicate files before attempting to re-run the sync.\n\n"
+        + "\n".join(duplicated_files_messages)
+    )
+
+    return error_message

airbyte_cdk/sources/file_based/file_based_source.py

@@ -242,7 +242,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         stream=self._make_default_stream(
                             stream_config=stream_config,
                             cursor=cursor,
-                            use_file_transfer=self._use_file_transfer(parsed_config),
+                            parsed_config=parsed_config,
                         ),
                         source=self,
                         logger=self.logger,
@@ -273,7 +273,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         stream=self._make_default_stream(
                             stream_config=stream_config,
                             cursor=cursor,
-                            use_file_transfer=self._use_file_transfer(parsed_config),
+                            parsed_config=parsed_config,
                         ),
                         source=self,
                         logger=self.logger,
@@ -285,7 +285,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                     stream = self._make_default_stream(
                         stream_config=stream_config,
                         cursor=cursor,
-                        use_file_transfer=self._use_file_transfer(parsed_config),
+                        parsed_config=parsed_config,
                     )
 
                 streams.append(stream)
@@ -298,7 +298,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         self,
         stream_config: FileBasedStreamConfig,
         cursor: Optional[AbstractFileBasedCursor],
-        use_file_transfer: bool = False,
+        parsed_config: AbstractFileBasedSpec,
     ) -> AbstractFileBasedStream:
         return DefaultFileBasedStream(
             config=stream_config,
@@ -310,7 +310,10 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             validation_policy=self._validate_and_get_validation_policy(stream_config),
             errors_collector=self.errors_collector,
             cursor=cursor,
-            use_file_transfer=use_file_transfer,
+            use_file_transfer=self._use_file_transfer(parsed_config),
+            preserve_subdirectories_directories=self._preserve_subdirectories_directories(
+                parsed_config
+            ),
         )
 
     def _get_stream_from_catalog(
@@ -385,3 +388,14 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             and parsed_config.delivery_method.delivery_type == "use_file_transfer"
         )
         return use_file_transfer
+
+    @staticmethod
+    def _preserve_subdirectories_directories(parsed_config: AbstractFileBasedSpec) -> bool:
+        # fall back to preserve subdirectories if config is not present or incomplete
+        if (
+            hasattr(parsed_config, "delivery_options")
+            and parsed_config.delivery_options is not None
+            and hasattr(parsed_config.delivery_options, "preserve_subdirectories_directories")
+        ):
+            return parsed_config.delivery_options.preserve_subdirectories_directories
+        return True

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -135,6 +135,17 @@ class AbstractFileBasedStreamReader(ABC):
             return use_file_transfer
         return False
 
+    def preserve_subdirectories_directories(self) -> bool:
+        # fall back to preserve subdirectories if config is not present or incomplete
+        if (
+            self.config
+            and hasattr(self.config, "delivery_options")
+            and self.config.delivery_options is not None
+            and hasattr(self.config.delivery_options, "preserve_subdirectories_directories")
+        ):
+            return self.config.delivery_options.preserve_subdirectories_directories
+        return True
+
     @abstractmethod
     def get_file(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
@@ -159,10 +170,13 @@ class AbstractFileBasedStreamReader(ABC):
         """
         ...
 
-    @staticmethod
-    def _get_file_transfer_paths(file: RemoteFile, local_directory: str) -> List[str]:
-        # Remove left slashes from source path format to make relative path for writing locally
-        file_relative_path = file.uri.lstrip("/")
+    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
+        preserve_subdirectories_directories = self.preserve_subdirectories_directories()
+        if preserve_subdirectories_directories:
+            # Remove left slashes from source path format to make relative path for writing locally
+            file_relative_path = file.uri.lstrip("/")
+        else:
+            file_relative_path = path.basename(file.uri)
         local_file_path = path.join(local_directory, file_relative_path)
 
         # Ensure the local directory exists

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -5,20 +5,24 @@
 import asyncio
 import itertools
 import traceback
+from collections import defaultdict
 from copy import deepcopy
 from functools import cache
-from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Union
+from os import path
+from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.exceptions import (
+    DuplicatedFilesError,
     FileBasedSourceError,
     InvalidSchemaError,
     MissingSchemaError,
     RecordParseError,
     SchemaInferenceError,
     StopSyncPerValidationPolicy,
+    format_duplicate_files_error_message,
 )
 from airbyte_cdk.sources.file_based.file_types import FileTransfer
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
@@ -43,6 +47,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     """
 
     FILE_TRANSFER_KW = "use_file_transfer"
+    PRESERVE_SUBDIRECTORIES_KW = "preserve_subdirectories_directories"
+    FILES_KEY = "files"
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
     ab_last_mod_col = "_ab_source_file_last_modified"
     ab_file_name_col = "_ab_source_file_url"
@@ -50,10 +56,14 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     source_file_url = "source_file_url"
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
     use_file_transfer = False
+    preserve_subdirectories_directories = True
 
     def __init__(self, **kwargs: Any):
         if self.FILE_TRANSFER_KW in kwargs:
             self.use_file_transfer = kwargs.pop(self.FILE_TRANSFER_KW, False)
+            self.preserve_subdirectories_directories = kwargs.pop(
+                self.PRESERVE_SUBDIRECTORIES_KW, True
+            )
         super().__init__(**kwargs)
 
     @property
@@ -98,15 +108,43 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
+    def _duplicated_files_names(
+        self, slices: List[dict[str, List[RemoteFile]]]
+    ) -> List[dict[str, List[str]]]:
+        seen_file_names = set()
+        duplicates_file_names = set()
+        file_paths = defaultdict(list)
+        for file_slice in slices:
+            for file_found in file_slice[self.FILES_KEY]:
+                file_name = path.basename(file_found.uri)
+                if file_name not in seen_file_names:
+                    seen_file_names.add(file_name)
+                else:
+                    duplicates_file_names.add(file_name)
+                file_paths[file_name].append(file_found.uri)
+        return [
+            {duplicated_file: file_paths[duplicated_file]}
+            for duplicated_file in duplicates_file_names
+        ]
+
     def compute_slices(self) -> Iterable[Optional[Mapping[str, Any]]]:
         # Sort files by last_modified, uri and return them grouped by last_modified
         all_files = self.list_files()
         files_to_read = self._cursor.get_files_to_sync(all_files, self.logger)
         sorted_files_to_read = sorted(files_to_read, key=lambda f: (f.last_modified, f.uri))
         slices = [
-            {"files": list(group[1])}
+            {self.FILES_KEY: list(group[1])}
             for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)
         ]
+        if slices and not self.preserve_subdirectories_directories:
+            duplicated_files_names = self._duplicated_files_names(slices)
+            if duplicated_files_names:
+                raise DuplicatedFilesError(
+                    format_duplicate_files_error_message(
+                        stream_name=self.name, duplicated_files_names=duplicated_files_names
+                    ),
+                    stream=self.name,
+                )
         return slices
 
     def transform_record(

{airbyte_cdk-6.13.0.dist-info → airbyte_cdk-6.13.1.dev4100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.13.0
+Version: 6.13.1.dev4100
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.13.0.dist-info → airbyte_cdk-6.13.1.dev4100.dist-info}/RECORD

@@ -140,10 +140,10 @@ airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=-P
 airbyte_cdk/sources/declarative/requesters/paginators/paginator.py,sha256=ZgyvH7DOrASQ5K__J5SRAXH3REUW2n3yPHnFW9xq4NU,1972
 airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py,sha256=2gly8fuZpDNwtu1Qg6oE2jBLGqQRdzSLJdnpk_iDV6I,767
 airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py,sha256=vFzpNv8BdgXrYO5qhi2_Un4x4y-EAQWxinZtEPWz5KI,3654
-airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py,sha256=TKG4Mp1t8MfmFJDeHtXmxCp_ibRK03J5O04N5HVtBvE,3430
+airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py,sha256=pMPi6iQrhtrI9VRPj218QNM_OqD8lX8P3Tr9yloKoE8,3367
 airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py,sha256=kQGpfr-dOwarxTIf2S4sHVulBzm8zSwQXBM7rOhkafA,2491
 airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py,sha256=ABpO4t0UUziBZnyml8UT_NhlF6loekhQji57TpKnaiY,1290
-airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py,sha256=2b005ulACvHgIL8ktTWwposu4umowyu0iGV2mGOb_Tg,2290
+airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py,sha256=-8NwokW-aKwv8DdeHh1ssODTobBYSOmIhH2-IjSjlNA,2213
 airbyte_cdk/sources/declarative/requesters/request_option.py,sha256=_qmv8CLQQ3fERt6BuMZeRu6tZXscPoeARx1VJdWMQ_M,1055
 airbyte_cdk/sources/declarative/requesters/request_options/__init__.py,sha256=WCwpKqM4wKqy-DHJaCHbKAlFqRVOqMi9K5qonxIfi_Y,809
 airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py,sha256=FLkg0uzC9bc-zFnALWr0FLYpKsz8iK2xQsd4UOyeW08,3706
@@ -193,7 +193,7 @@ airbyte_cdk/sources/file_based/availability_strategy/__init__.py,sha256=ddKQfUmk
 airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py,sha256=01Nd4b7ERAbp-OZo_8rrAzFXWPTMwr02SnWiN17nx8Q,2363
 airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py,sha256=j9T5TimfWFUz7nqsaj-83G3xWmDpsmeSbDnaUNmz0UM,5849
 airbyte_cdk/sources/file_based/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py,sha256=tj-M1L5BTa5yIQ3jHo09CtCTSq_eR-68zgyOPqwsurw,6455
+airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py,sha256=1V7NpVdcVA7QNF_mI0uH5X3WqBvrS6U-W1iHoTiEDtY,6926
 airbyte_cdk/sources/file_based/config/avro_format.py,sha256=NxTF96ewzn6HuhgodsY7Rpb-ybr1ZEWW5d4Vid64g5A,716
 airbyte_cdk/sources/file_based/config/csv_format.py,sha256=NWekkyT8dTwiVK0mwa_krQD4FJPHSDfILo8kPAg3-Vs,8006
 airbyte_cdk/sources/file_based/config/excel_format.py,sha256=9qAmTsT6SoVzNfNv0oBVkVCmiyqQuVAbfRKajjoa7Js,378
@@ -204,9 +204,9 @@ airbyte_cdk/sources/file_based/config/unstructured_format.py,sha256=tIbB9Pn1HqU6
 airbyte_cdk/sources/file_based/discovery_policy/__init__.py,sha256=gl3ey6mZbyfraB9P3pFhf9UJp2JeTZ1SUFAopy2iBvY,301
 airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py,sha256=dCfXX529Rd5rtopg4VeEgTPJjFtqjtjzPq6LCw18Wt0,605
 airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py,sha256=-xujTidtrq6HC00WKbjQh1CZdT5LMuzkp5BLjqDmfTY,1007
-airbyte_cdk/sources/file_based/exceptions.py,sha256=AEELNIRzKPX6eopKd_2jhE7WiNeR0Aw7nQWVOL8fvkc,5760
-airbyte_cdk/sources/file_based/file_based_source.py,sha256=RfpctRNLJ_EHKKEc2E1EZGYRfhG0Z9o6TgsKS4XrSNY,16652
-airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=ohxKlqPuV7TGwjyRy_gaWUol8QN5lBSoCYoaqBtRh1c,6179
+airbyte_cdk/sources/file_based/exceptions.py,sha256=KfOgQgssBKgsv3h5po2IG1DhZcH664Zf_fx96mBlPSg,6761
+airbyte_cdk/sources/file_based/file_based_source.py,sha256=Z-1EQS8_gczgBgBHJV75LCwHOtXgonEqxNpXmNmqVTM,17263
+airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=Uwg_e9wHdJ2UaWxHcW2Ph6S4sHDhB0_MoyCEW7neVdA,6888
 airbyte_cdk/sources/file_based/file_types/__init__.py,sha256=blCLn0-2LC-ZdgcNyDEhqM2RiUvEjEBh-G4-t32ZtuM,1268
 airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=XNx-JC-sgzH9u3nOJ2M59FxBXvtig8LN6BIkeDOavZA,10858
 airbyte_cdk/sources/file_based/file_types/csv_parser.py,sha256=QlCXB-ry3np67Q_VerQEPoWDOTcPTB6Go4ydZxY9ae4,20445
@@ -232,7 +232,7 @@ airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_c
 airbyte_cdk/sources/file_based/stream/cursor/__init__.py,sha256=MhFB5hOo8sjwvCh8gangaymdg3EJWYt_72brFOZt068,191
 airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py,sha256=om-x3gZFPgWDpi15S9RxZmR36VHnk8sytgN6LlBQhAw,1934
 airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py,sha256=VGV7xLyBribuBMVrXtO1xqkWJD86bl7yhXtjnwLMohM,7051
-airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=rpwU6AOyhFLuXtcFKkcOHFWbRQ4kLCOKzAjcID_M87k,16770
+airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=HRjR0rQGc8cYK2PxpLgAvZQ--jvtV8QgS1QIxkemnko,18413
 airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
 airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
 airbyte_cdk/sources/http_logger.py,sha256=TyBmtRA6D9g0XDkKGvdM415b36RXDjgfkwRewDsH8-0,1576
@@ -340,8 +340,8 @@ airbyte_cdk/utils/slice_hasher.py,sha256=-pHexlNYoWYPnXNH-M7HEbjmeJe9Zk7SJijdQ7d
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.13.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.13.0.dist-info/METADATA,sha256=Iwa16bm84hAFYmkLc2CvpiktJh7jFymHCYFUfoPUZlI,5988
-airbyte_cdk-6.13.0.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-airbyte_cdk-6.13.0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.13.0.dist-info/RECORD,,
+airbyte_cdk-6.13.1.dev4100.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.13.1.dev4100.dist-info/METADATA,sha256=P_PnklFtJqLdDdAxsiOLX7Ydkee8d7jz9m_v2XNCRAI,5996
+airbyte_cdk-6.13.1.dev4100.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+airbyte_cdk-6.13.1.dev4100.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.13.1.dev4100.dist-info/RECORD,,