airbyte-cdk 6.13.1.dev41012__tar.gz → 6.14.0__tar.gz

{airbyte_cdk-6.13.1.dev41012 → airbyte_cdk-6.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.13.1.dev41012
+Version: 6.14.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.13.1.dev41012 → airbyte_cdk-6.14.0}/airbyte_cdk/sources/declarative/interpolation/macros.py

@@ -94,6 +94,26 @@ def max(*args: typing.Any) -> typing.Any:
     return builtins.max(*args)
 
 
+def min(*args: typing.Any) -> typing.Any:
+    """
+    Returns smallest object of an iterable, or two or more arguments.
+
+    min(iterable, *[, default=obj, key=func]) -> value
+    min(arg1, arg2, *args, *[, key=func]) -> value
+
+    Usage:
+    `"{{ min(2,3) }}"
+
+    With a single iterable argument, return its smallest item. The
+    default keyword-only argument specifies an object to return if
+    the provided iterable is empty.
+    With two or more arguments, return the smallest argument.
+    :param args: args to compare
+    :return: smallest argument
+    """
+    return builtins.min(*args)
+
+
 def day_delta(num_days: int, format: str = "%Y-%m-%dT%H:%M:%S.%f%z") -> str:
     """
     Returns datetime of now() + num_days
@@ -147,6 +167,7 @@ _macros_list = [
     today_utc,
     timestamp,
     max,
+    min,
     day_delta,
     duration,
     format_datetime,

{airbyte_cdk-6.13.1.dev41012 → airbyte_cdk-6.14.0}/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py

@@ -83,7 +83,9 @@ class OffsetIncrement(PaginationStrategy):
             return self._offset
 
     def reset(self, reset_value: Optional[Any] = 0) -> None:
-        if not isinstance(reset_value, int):
+        if reset_value is None:
+            self._offset = 0
+        elif not isinstance(reset_value, int):
             raise ValueError(
                 f"Reset value {reset_value} for OffsetIncrement pagination strategy was not an integer"
             )

{airbyte_cdk-6.13.1.dev41012 → airbyte_cdk-6.14.0}/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py

@@ -53,7 +53,10 @@ class StopConditionPaginationStrategyDecorator(PaginationStrategy):
         return self._delegate.next_page_token(response, last_page_size, last_record)
 
     def reset(self, reset_value: Optional[Any] = None) -> None:
-        self._delegate.reset(reset_value)
+        if reset_value:
+            self._delegate.reset(reset_value)
+        else:
+            self._delegate.reset()
 
     def get_page_size(self) -> Optional[int]:
         return self._delegate.get_page_size()

{airbyte_cdk-6.13.1.dev41012 → airbyte_cdk-6.14.0}/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py

@@ -31,12 +31,6 @@ class DeliverRawFiles(BaseModel):
 
     delivery_type: Literal["use_file_transfer"] = Field("use_file_transfer", const=True)
 
-    preserve_subdirectories_directories: bool = Field(
-        title="Preserve Subdirectories in File Paths",
-        description="If enabled replicate source folder structure",
-        default=True,
-    )
-
 
 class AbstractFileBasedSpec(BaseModel):
     """

{airbyte_cdk-6.13.1.dev41012 → airbyte_cdk-6.14.0}/airbyte_cdk/sources/file_based/exceptions.py

@@ -111,10 +111,6 @@ class ErrorListingFiles(BaseFileBasedSourceError):
     pass
 
 
-class DuplicatedFilesError(BaseFileBasedSourceError):
-    pass
-
-
 class CustomFileBasedException(AirbyteTracedException):
     """
     A specialized exception for file-based connectors.
@@ -127,25 +123,3 @@ class CustomFileBasedException(AirbyteTracedException):
 
 class FileSizeLimitError(CustomFileBasedException):
     pass
-
-
-def format_duplicate_files_error_message(
-    stream_name: str, duplicated_files_names: List[dict[str, List[str]]]
-) -> str:
-    duplicated_files_messages = []
-    for duplicated_file in duplicated_files_names:
-        for duplicated_file_name, file_paths in duplicated_file.items():
-            file_duplicated_message = (
-                f"{len(file_paths)} duplicates found for file name {duplicated_file_name}:\n\n"
-                + "".join(f"\n - {file_paths}")
-            )
-            duplicated_files_messages.append(file_duplicated_message)
-
-    error_message = (
-        f"ERROR: Duplicate filenames found for stream {stream_name}. "
-        "Duplicate file names are not allowed if the Preserve Subdirectories in File Paths option is disabled. "
-        "Please remove or rename the duplicate files before attempting to re-run the sync.\n\n"
-        + "\n".join(duplicated_files_messages)
-    )
-
-    return error_message

{airbyte_cdk-6.13.1.dev41012 → airbyte_cdk-6.14.0}/airbyte_cdk/sources/file_based/file_based_source.py

@@ -242,7 +242,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         stream=self._make_default_stream(
                             stream_config=stream_config,
                             cursor=cursor,
-                            parsed_config=parsed_config,
+                            use_file_transfer=self._use_file_transfer(parsed_config),
                         ),
                         source=self,
                         logger=self.logger,
@@ -273,7 +273,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         stream=self._make_default_stream(
                             stream_config=stream_config,
                             cursor=cursor,
-                            parsed_config=parsed_config,
+                            use_file_transfer=self._use_file_transfer(parsed_config),
                         ),
                         source=self,
                         logger=self.logger,
@@ -285,7 +285,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                     stream = self._make_default_stream(
                         stream_config=stream_config,
                         cursor=cursor,
-                        parsed_config=parsed_config,
+                        use_file_transfer=self._use_file_transfer(parsed_config),
                     )
 
                 streams.append(stream)
@@ -298,7 +298,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         self,
         stream_config: FileBasedStreamConfig,
         cursor: Optional[AbstractFileBasedCursor],
-        parsed_config: AbstractFileBasedSpec,
+        use_file_transfer: bool = False,
     ) -> AbstractFileBasedStream:
         return DefaultFileBasedStream(
             config=stream_config,
@@ -310,10 +310,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             validation_policy=self._validate_and_get_validation_policy(stream_config),
             errors_collector=self.errors_collector,
             cursor=cursor,
-            use_file_transfer=self._use_file_transfer(parsed_config),
-            preserve_subdirectories_directories=self._preserve_subdirectories_directories(
-                parsed_config
-            ),
+            use_file_transfer=use_file_transfer,
         )
 
     def _get_stream_from_catalog(
@@ -388,14 +385,3 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             and parsed_config.delivery_method.delivery_type == "use_file_transfer"
         )
         return use_file_transfer
-
-    @staticmethod
-    def _preserve_subdirectories_directories(parsed_config: AbstractFileBasedSpec) -> bool:
-        # fall back to preserve subdirectories if config is not present or incomplete
-        if (
-            FileBasedSource._use_file_transfer(parsed_config)
-            and hasattr(parsed_config.delivery_method, "preserve_subdirectories_directories")
-            and parsed_config.delivery_method.preserve_subdirectories_directories is not None
-        ):
-            return parsed_config.delivery_method.preserve_subdirectories_directories
-        return True

{airbyte_cdk-6.13.1.dev41012 → airbyte_cdk-6.14.0}/airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -135,16 +135,6 @@ class AbstractFileBasedStreamReader(ABC):
             return use_file_transfer
         return False
 
-    def preserve_subdirectories_directories(self) -> bool:
-        # fall back to preserve subdirectories if config is not present or incomplete
-        if (
-            self.use_file_transfer()
-            and hasattr(self.config.delivery_method, "preserve_subdirectories_directories")
-            and self.config.delivery_method.preserve_subdirectories_directories is not None
-        ):
-            return self.config.delivery_method.preserve_subdirectories_directories
-        return True
-
     @abstractmethod
     def get_file(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
@@ -169,13 +159,10 @@ class AbstractFileBasedStreamReader(ABC):
         """
         ...
 
-    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
-        preserve_subdirectories_directories = self.preserve_subdirectories_directories()
-        if preserve_subdirectories_directories:
-            # Remove left slashes from source path format to make relative path for writing locally
-            file_relative_path = file.uri.lstrip("/")
-        else:
-            file_relative_path = path.basename(file.uri)
+    @staticmethod
+    def _get_file_transfer_paths(file: RemoteFile, local_directory: str) -> List[str]:
+        # Remove left slashes from source path format to make relative path for writing locally
+        file_relative_path = file.uri.lstrip("/")
         local_file_path = path.join(local_directory, file_relative_path)
 
         # Ensure the local directory exists

{airbyte_cdk-6.13.1.dev41012 → airbyte_cdk-6.14.0}/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -5,24 +5,20 @@
 import asyncio
 import itertools
 import traceback
-from collections import defaultdict
 from copy import deepcopy
 from functools import cache
-from os import path
-from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
+from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Union
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.exceptions import (
-    DuplicatedFilesError,
     FileBasedSourceError,
     InvalidSchemaError,
     MissingSchemaError,
     RecordParseError,
     SchemaInferenceError,
     StopSyncPerValidationPolicy,
-    format_duplicate_files_error_message,
 )
 from airbyte_cdk.sources.file_based.file_types import FileTransfer
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
@@ -47,8 +43,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     """
 
     FILE_TRANSFER_KW = "use_file_transfer"
-    PRESERVE_SUBDIRECTORIES_KW = "preserve_subdirectories_directories"
-    FILES_KEY = "files"
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
     ab_last_mod_col = "_ab_source_file_last_modified"
     ab_file_name_col = "_ab_source_file_url"
@@ -56,14 +50,10 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     source_file_url = "source_file_url"
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
     use_file_transfer = False
-    preserve_subdirectories_directories = True
 
     def __init__(self, **kwargs: Any):
         if self.FILE_TRANSFER_KW in kwargs:
             self.use_file_transfer = kwargs.pop(self.FILE_TRANSFER_KW, False)
-            self.preserve_subdirectories_directories = kwargs.pop(
-                self.PRESERVE_SUBDIRECTORIES_KW, True
-            )
         super().__init__(**kwargs)
 
     @property
@@ -108,43 +98,15 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
-    def _duplicated_files_names(
-        self, slices: List[dict[str, List[RemoteFile]]]
-    ) -> List[dict[str, List[str]]]:
-        seen_file_names = set()
-        duplicates_file_names = set()
-        file_paths = defaultdict(list)
-        for file_slice in slices:
-            for file_found in file_slice[self.FILES_KEY]:
-                file_name = path.basename(file_found.uri)
-                if file_name not in seen_file_names:
-                    seen_file_names.add(file_name)
-                else:
-                    duplicates_file_names.add(file_name)
-                file_paths[file_name].append(file_found.uri)
-        return [
-            {duplicated_file: file_paths[duplicated_file]}
-            for duplicated_file in duplicates_file_names
-        ]
-
     def compute_slices(self) -> Iterable[Optional[Mapping[str, Any]]]:
         # Sort files by last_modified, uri and return them grouped by last_modified
         all_files = self.list_files()
         files_to_read = self._cursor.get_files_to_sync(all_files, self.logger)
         sorted_files_to_read = sorted(files_to_read, key=lambda f: (f.last_modified, f.uri))
         slices = [
-            {self.FILES_KEY: list(group[1])}
+            {"files": list(group[1])}
             for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)
         ]
-        if slices and not self.preserve_subdirectories_directories:
-            duplicated_files_names = self._duplicated_files_names(slices)
-            if duplicated_files_names:
-                raise DuplicatedFilesError(
-                    format_duplicate_files_error_message(
-                        stream_name=self.name, duplicated_files_names=duplicated_files_names
-                    ),
-                    stream=self.name,
-                )
         return slices
 
     def transform_record(

{airbyte_cdk-6.13.1.dev41012 → airbyte_cdk-6.14.0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.13.1.dev041012"  # Version will be calculated dynamically.
+version = "6.14.0"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false