airbyte-cdk 6.13.0__tar.gz → 6.13.1.dev4100__tar.gz

{airbyte_cdk-6.13.0 → airbyte_cdk-6.13.1.dev4100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.13.0
+Version: 6.13.1.dev4100
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.13.0 → airbyte_cdk-6.13.1.dev4100}/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py

@@ -83,9 +83,7 @@ class OffsetIncrement(PaginationStrategy):
             return self._offset
 
     def reset(self, reset_value: Optional[Any] = 0) -> None:
-        if reset_value is None:
-            self._offset = 0
-        elif not isinstance(reset_value, int):
+        if not isinstance(reset_value, int):
             raise ValueError(
                 f"Reset value {reset_value} for OffsetIncrement pagination strategy was not an integer"
             )

{airbyte_cdk-6.13.0 → airbyte_cdk-6.13.1.dev4100}/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py

@@ -53,10 +53,7 @@ class StopConditionPaginationStrategyDecorator(PaginationStrategy):
         return self._delegate.next_page_token(response, last_page_size, last_record)
 
     def reset(self, reset_value: Optional[Any] = None) -> None:
-        if reset_value:
-            self._delegate.reset(reset_value)
-        else:
-            self._delegate.reset()
+        self._delegate.reset(reset_value)
 
     def get_page_size(self) -> Optional[int]:
         return self._delegate.get_page_size()

{airbyte_cdk-6.13.0 → airbyte_cdk-6.13.1.dev4100}/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py

@@ -14,6 +14,13 @@ from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileB
 from airbyte_cdk.sources.utils import schema_helpers
 
 
+class DeliveryOptions(BaseModel):
+    preserve_subdirectories_directories: bool = Field(
+        True,
+        description="Flag indicating we should preserve subdirectories directories",
+    )
+
+
 class DeliverRecords(BaseModel):
     class Config(OneOfOptionConfig):
         title = "Replicate Records"
@@ -30,6 +37,11 @@ class DeliverRawFiles(BaseModel):
         discriminator = "delivery_type"
 
     delivery_type: Literal["use_file_transfer"] = Field("use_file_transfer", const=True)
+    delivery_options: Optional[DeliveryOptions] = Field(
+        title="Delivery Options",
+        type="object",
+        order=2,
+    )
 
 
 class AbstractFileBasedSpec(BaseModel):
@@ -65,6 +77,12 @@ class AbstractFileBasedSpec(BaseModel):
         airbyte_hidden=True,
     )
 
+    delivery_options: Optional[DeliveryOptions] = Field(
+        title="Delivery Options",
+        type="object",
+        order=8,
+    )
+
     @classmethod
     @abstractmethod
     def documentation_url(cls) -> AnyUrl:

{airbyte_cdk-6.13.0 → airbyte_cdk-6.13.1.dev4100}/airbyte_cdk/sources/file_based/exceptions.py

@@ -111,6 +111,10 @@ class ErrorListingFiles(BaseFileBasedSourceError):
     pass
 
 
+class DuplicatedFilesError(BaseFileBasedSourceError):
+    pass
+
+
 class CustomFileBasedException(AirbyteTracedException):
     """
     A specialized exception for file-based connectors.
@@ -123,3 +127,25 @@ class CustomFileBasedException(AirbyteTracedException):
 
 class FileSizeLimitError(CustomFileBasedException):
     pass
+
+
+def format_duplicate_files_error_message(
+    stream_name: str, duplicated_files_names: List[dict[str, List[str]]]
+) -> str:
+    duplicated_files_messages = []
+    for duplicated_file in duplicated_files_names:
+        for duplicated_file_name, file_paths in duplicated_file.items():
+            file_duplicated_message = (
+                f"{len(file_paths)} duplicates found for file name {duplicated_file_name}:\n\n"
+                + "".join(f"\n - {file_paths}")
+            )
+            duplicated_files_messages.append(file_duplicated_message)
+
+    error_message = (
+        f"ERROR: Duplicate filenames found for stream {stream_name}. "
+        "Duplicate file names are not allowed if the Preserve Subdirectories in File Paths option is disabled. "
+        "Please remove or rename the duplicate files before attempting to re-run the sync.\n\n"
+        + "\n".join(duplicated_files_messages)
+    )
+
+    return error_message

{airbyte_cdk-6.13.0 → airbyte_cdk-6.13.1.dev4100}/airbyte_cdk/sources/file_based/file_based_source.py

@@ -242,7 +242,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         stream=self._make_default_stream(
                             stream_config=stream_config,
                             cursor=cursor,
-                            use_file_transfer=self._use_file_transfer(parsed_config),
+                            parsed_config=parsed_config,
                         ),
                         source=self,
                         logger=self.logger,
@@ -273,7 +273,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         stream=self._make_default_stream(
                             stream_config=stream_config,
                             cursor=cursor,
-                            use_file_transfer=self._use_file_transfer(parsed_config),
+                            parsed_config=parsed_config,
                         ),
                         source=self,
                         logger=self.logger,
@@ -285,7 +285,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                     stream = self._make_default_stream(
                         stream_config=stream_config,
                         cursor=cursor,
-                        use_file_transfer=self._use_file_transfer(parsed_config),
+                        parsed_config=parsed_config,
                     )
 
                 streams.append(stream)
@@ -298,7 +298,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         self,
         stream_config: FileBasedStreamConfig,
         cursor: Optional[AbstractFileBasedCursor],
-        use_file_transfer: bool = False,
+        parsed_config: AbstractFileBasedSpec,
     ) -> AbstractFileBasedStream:
         return DefaultFileBasedStream(
             config=stream_config,
@@ -310,7 +310,10 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             validation_policy=self._validate_and_get_validation_policy(stream_config),
             errors_collector=self.errors_collector,
             cursor=cursor,
-            use_file_transfer=use_file_transfer,
+            use_file_transfer=self._use_file_transfer(parsed_config),
+            preserve_subdirectories_directories=self._preserve_subdirectories_directories(
+                parsed_config
+            ),
         )
 
     def _get_stream_from_catalog(
@@ -385,3 +388,14 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             and parsed_config.delivery_method.delivery_type == "use_file_transfer"
         )
         return use_file_transfer
+
+    @staticmethod
+    def _preserve_subdirectories_directories(parsed_config: AbstractFileBasedSpec) -> bool:
+        # fall back to preserve subdirectories if config is not present or incomplete
+        if (
+            hasattr(parsed_config, "delivery_options")
+            and parsed_config.delivery_options is not None
+            and hasattr(parsed_config.delivery_options, "preserve_subdirectories_directories")
+        ):
+            return parsed_config.delivery_options.preserve_subdirectories_directories
+        return True

{airbyte_cdk-6.13.0 → airbyte_cdk-6.13.1.dev4100}/airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -135,6 +135,17 @@ class AbstractFileBasedStreamReader(ABC):
             return use_file_transfer
         return False
 
+    def preserve_subdirectories_directories(self) -> bool:
+        # fall back to preserve subdirectories if config is not present or incomplete
+        if (
+            self.config
+            and hasattr(self.config, "delivery_options")
+            and self.config.delivery_options is not None
+            and hasattr(self.config.delivery_options, "preserve_subdirectories_directories")
+        ):
+            return self.config.delivery_options.preserve_subdirectories_directories
+        return True
+
     @abstractmethod
     def get_file(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
@@ -159,10 +170,13 @@ class AbstractFileBasedStreamReader(ABC):
         """
         ...
 
-    @staticmethod
-    def _get_file_transfer_paths(file: RemoteFile, local_directory: str) -> List[str]:
-        # Remove left slashes from source path format to make relative path for writing locally
-        file_relative_path = file.uri.lstrip("/")
+    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
+        preserve_subdirectories_directories = self.preserve_subdirectories_directories()
+        if preserve_subdirectories_directories:
+            # Remove left slashes from source path format to make relative path for writing locally
+            file_relative_path = file.uri.lstrip("/")
+        else:
+            file_relative_path = path.basename(file.uri)
         local_file_path = path.join(local_directory, file_relative_path)
 
         # Ensure the local directory exists

{airbyte_cdk-6.13.0 → airbyte_cdk-6.13.1.dev4100}/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -5,20 +5,24 @@
 import asyncio
 import itertools
 import traceback
+from collections import defaultdict
 from copy import deepcopy
 from functools import cache
-from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Union
+from os import path
+from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.exceptions import (
+    DuplicatedFilesError,
     FileBasedSourceError,
     InvalidSchemaError,
     MissingSchemaError,
     RecordParseError,
     SchemaInferenceError,
     StopSyncPerValidationPolicy,
+    format_duplicate_files_error_message,
 )
 from airbyte_cdk.sources.file_based.file_types import FileTransfer
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
@@ -43,6 +47,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     """
 
     FILE_TRANSFER_KW = "use_file_transfer"
+    PRESERVE_SUBDIRECTORIES_KW = "preserve_subdirectories_directories"
+    FILES_KEY = "files"
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
     ab_last_mod_col = "_ab_source_file_last_modified"
     ab_file_name_col = "_ab_source_file_url"
@@ -50,10 +56,14 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     source_file_url = "source_file_url"
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
     use_file_transfer = False
+    preserve_subdirectories_directories = True
 
     def __init__(self, **kwargs: Any):
         if self.FILE_TRANSFER_KW in kwargs:
             self.use_file_transfer = kwargs.pop(self.FILE_TRANSFER_KW, False)
+            self.preserve_subdirectories_directories = kwargs.pop(
+                self.PRESERVE_SUBDIRECTORIES_KW, True
+            )
         super().__init__(**kwargs)
 
     @property
@@ -98,15 +108,43 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
+    def _duplicated_files_names(
+        self, slices: List[dict[str, List[RemoteFile]]]
+    ) -> List[dict[str, List[str]]]:
+        seen_file_names = set()
+        duplicates_file_names = set()
+        file_paths = defaultdict(list)
+        for file_slice in slices:
+            for file_found in file_slice[self.FILES_KEY]:
+                file_name = path.basename(file_found.uri)
+                if file_name not in seen_file_names:
+                    seen_file_names.add(file_name)
+                else:
+                    duplicates_file_names.add(file_name)
+                file_paths[file_name].append(file_found.uri)
+        return [
+            {duplicated_file: file_paths[duplicated_file]}
+            for duplicated_file in duplicates_file_names
+        ]
+
     def compute_slices(self) -> Iterable[Optional[Mapping[str, Any]]]:
         # Sort files by last_modified, uri and return them grouped by last_modified
         all_files = self.list_files()
         files_to_read = self._cursor.get_files_to_sync(all_files, self.logger)
         sorted_files_to_read = sorted(files_to_read, key=lambda f: (f.last_modified, f.uri))
         slices = [
-            {"files": list(group[1])}
+            {self.FILES_KEY: list(group[1])}
             for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)
         ]
+        if slices and not self.preserve_subdirectories_directories:
+            duplicated_files_names = self._duplicated_files_names(slices)
+            if duplicated_files_names:
+                raise DuplicatedFilesError(
+                    format_duplicate_files_error_message(
+                        stream_name=self.name, duplicated_files_names=duplicated_files_names
+                    ),
+                    stream=self.name,
+                )
         return slices
 
     def transform_record(

{airbyte_cdk-6.13.0 → airbyte_cdk-6.13.1.dev4100}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.13.0"  # Version will be calculated dynamically.
+version = "6.13.1.dev04100"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false