airbyte-cdk 6.13.1.dev41012__py3-none-any.whl → 6.14.0.dev1__py3-none-any.whl

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -3,7 +3,7 @@
 #
 
 import logging
-from typing import Any, Callable, Generic, Iterator, List, Mapping, Optional, Tuple, Union
+from typing import Any, Generic, Iterator, List, Mapping, Optional, Tuple
 
 from airbyte_cdk.models import (
     AirbyteCatalog,
@@ -28,15 +28,11 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     DatetimeBasedCursor as DatetimeBasedCursorModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    DeclarativeStream as DeclarativeStreamModel,
-)
 from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
-    ComponentDefinition,
     ModelToComponentFactory,
 )
 from airbyte_cdk.sources.declarative.requesters import HttpRequester
-from airbyte_cdk.sources.declarative.retrievers import Retriever, SimpleRetriever
+from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever
 from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import (
     DeclarativePartitionFactory,
     StreamSlicerPartitionGenerator,
@@ -52,7 +48,6 @@ from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
 from airbyte_cdk.sources.streams.concurrent.cursor import FinalStateCursor
 from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream
-from airbyte_cdk.sources.types import Config, StreamState
 
 
 class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
@@ -194,10 +189,11 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             # Some low-code sources use a combination of DeclarativeStream and regular Python streams. We can't inspect
             # these legacy Python streams the way we do low-code streams to determine if they are concurrent compatible,
             # so we need to treat them as synchronous
-            if (
-                isinstance(declarative_stream, DeclarativeStream)
-                and name_to_stream_mapping[declarative_stream.name]["retriever"]["type"]
+            if isinstance(declarative_stream, DeclarativeStream) and (
+                name_to_stream_mapping[declarative_stream.name]["retriever"]["type"]
                 == "SimpleRetriever"
+                or name_to_stream_mapping[declarative_stream.name]["retriever"]["type"]
+                == "AsyncRetriever"
             ):
                 incremental_sync_component_definition = name_to_stream_mapping[
                     declarative_stream.name
@@ -217,6 +213,11 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                     and not incremental_sync_component_definition
                 )
 
+                is_async_job_stream = (
+                    name_to_stream_mapping[declarative_stream.name].get("retriever", {}).get("type")
+                    == "AsyncRetriever"
+                )
+
                 if self._is_datetime_incremental_without_partition_routing(
                     declarative_stream, incremental_sync_component_definition
                 ):
@@ -234,15 +235,25 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                         stream_state=stream_state,
                     )
 
+                    retriever = declarative_stream.retriever
+
+                    # This is an optimization so that we don't invoke any cursor or state management flows within the
+                    # low-code framework because state management is handled through the ConcurrentCursor.
+                    if declarative_stream and isinstance(retriever, SimpleRetriever):
+                        # Also a temporary hack. In the legacy Stream implementation, as part of the read,
+                        # set_initial_state() is called to instantiate incoming state on the cursor. Although we no
+                        # longer rely on the legacy low-code cursor for concurrent checkpointing, low-code components
+                        # like StopConditionPaginationStrategyDecorator and ClientSideIncrementalRecordFilterDecorator
+                        # still rely on a DatetimeBasedCursor that is properly initialized with state.
+                        if retriever.cursor:
+                            retriever.cursor.set_initial_state(stream_state=stream_state)
+                        retriever.cursor = None
+
                     partition_generator = StreamSlicerPartitionGenerator(
                         DeclarativePartitionFactory(
                             declarative_stream.name,
                             declarative_stream.get_json_schema(),
-                            self._retriever_factory(
-                                name_to_stream_mapping[declarative_stream.name],
-                                config,
-                                stream_state,
-                            ),
+                            retriever,
                             self.message_repository,
                         ),
                         cursor,
@@ -272,11 +283,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                         DeclarativePartitionFactory(
                             declarative_stream.name,
                             declarative_stream.get_json_schema(),
-                            self._retriever_factory(
-                                name_to_stream_mapping[declarative_stream.name],
-                                config,
-                                {},
-                            ),
+                            declarative_stream.retriever,
                             self.message_repository,
                         ),
                         declarative_stream.retriever.stream_slicer,
@@ -415,34 +422,3 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                 if stream.stream.name not in concurrent_stream_names
             ]
         )
-
-    def _retriever_factory(
-        self, stream_config: ComponentDefinition, source_config: Config, stream_state: StreamState
-    ) -> Callable[[], Retriever]:
-        def _factory_method() -> Retriever:
-            declarative_stream: DeclarativeStream = self._constructor.create_component(
-                DeclarativeStreamModel,
-                stream_config,
-                source_config,
-                emit_connector_builder_messages=self._emit_connector_builder_messages,
-            )
-
-            # This is an optimization so that we don't invoke any cursor or state management flows within the
-            # low-code framework because state management is handled through the ConcurrentCursor.
-            if (
-                declarative_stream
-                and declarative_stream.retriever
-                and isinstance(declarative_stream.retriever, SimpleRetriever)
-            ):
-                # Also a temporary hack. In the legacy Stream implementation, as part of the read, set_initial_state() is
-                # called to instantiate incoming state on the cursor. Although we no longer rely on the legacy low-code cursor
-                # for concurrent checkpointing, low-code components like StopConditionPaginationStrategyDecorator and
-                # ClientSideIncrementalRecordFilterDecorator still rely on a DatetimeBasedCursor that is properly initialized
-                # with state.
-                if declarative_stream.retriever.cursor:
-                    declarative_stream.retriever.cursor.set_initial_state(stream_state=stream_state)
-                declarative_stream.retriever.cursor = None
-
-            return declarative_stream.retriever
-
-        return _factory_method

airbyte_cdk/sources/declarative/interpolation/macros.py

@@ -94,6 +94,26 @@ def max(*args: typing.Any) -> typing.Any:
     return builtins.max(*args)
 
 
+def min(*args: typing.Any) -> typing.Any:
+    """
+    Returns smallest object of an iterable, or two or more arguments.
+
+    min(iterable, *[, default=obj, key=func]) -> value
+    min(arg1, arg2, *args, *[, key=func]) -> value
+
+    Usage:
+    `"{{ min(2,3) }}"
+
+    With a single iterable argument, return its smallest item. The
+    default keyword-only argument specifies an object to return if
+    the provided iterable is empty.
+    With two or more arguments, return the smallest argument.
+    :param args: args to compare
+    :return: smallest argument
+    """
+    return builtins.min(*args)
+
+
 def day_delta(num_days: int, format: str = "%Y-%m-%dT%H:%M:%S.%f%z") -> str:
     """
     Returns datetime of now() + num_days
@@ -147,6 +167,7 @@ _macros_list = [
     today_utc,
     timestamp,
     max,
+    min,
     day_delta,
     duration,
     format_datetime,

airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py

@@ -112,27 +112,39 @@ class DefaultPaginator(Paginator):
             )
         if isinstance(self.url_base, str):
             self.url_base = InterpolatedString(string=self.url_base, parameters=parameters)
-        self._token: Optional[Any] = self.pagination_strategy.initial_token
+
+    def get_initial_token(self) -> Optional[Any]:
+        """
+        Return the page token that should be used for the first request of a stream
+
+        WARNING: get_initial_token() should not be used by streams that use RFR that perform checkpointing
+        of state using page numbers. Because paginators are stateless
+        """
+        return self.pagination_strategy.initial_token
 
     def next_page_token(
-        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any] = None,
     ) -> Optional[Mapping[str, Any]]:
-        self._token = self.pagination_strategy.next_page_token(
-            response, last_page_size, last_record
+        next_page_token = self.pagination_strategy.next_page_token(
+            response=response,
+            last_page_size=last_page_size,
+            last_record=last_record,
+            last_page_token_value=last_page_token_value,
         )
-        if self._token:
-            return {"next_page_token": self._token}
+        if next_page_token:
+            return {"next_page_token": next_page_token}
         else:
             return None
 
-    def path(self) -> Optional[str]:
-        if (
-            self._token
-            and self.page_token_option
-            and isinstance(self.page_token_option, RequestPath)
-        ):
+    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+        token = next_page_token.get("next_page_token") if next_page_token else None
+        if token and self.page_token_option and isinstance(self.page_token_option, RequestPath):
             # Replace url base to only return the path
-            return str(self._token).replace(self.url_base.eval(self.config), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
+            return str(token).replace(self.url_base.eval(self.config), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
         else:
             return None
 
@@ -143,7 +155,7 @@ class DefaultPaginator(Paginator):
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> MutableMapping[str, Any]:
-        return self._get_request_options(RequestOptionType.request_parameter)
+        return self._get_request_options(RequestOptionType.request_parameter, next_page_token)
 
     def get_request_headers(
         self,
@@ -152,7 +164,7 @@ class DefaultPaginator(Paginator):
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, str]:
-        return self._get_request_options(RequestOptionType.header)
+        return self._get_request_options(RequestOptionType.header, next_page_token)
 
     def get_request_body_data(
         self,
@@ -161,7 +173,7 @@ class DefaultPaginator(Paginator):
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, Any]:
-        return self._get_request_options(RequestOptionType.body_data)
+        return self._get_request_options(RequestOptionType.body_data, next_page_token)
 
     def get_request_body_json(
         self,
@@ -170,25 +182,21 @@ class DefaultPaginator(Paginator):
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, Any]:
-        return self._get_request_options(RequestOptionType.body_json)
-
-    def reset(self, reset_value: Optional[Any] = None) -> None:
-        if reset_value:
-            self.pagination_strategy.reset(reset_value=reset_value)
-        else:
-            self.pagination_strategy.reset()
-        self._token = self.pagination_strategy.initial_token
+        return self._get_request_options(RequestOptionType.body_json, next_page_token)
 
-    def _get_request_options(self, option_type: RequestOptionType) -> MutableMapping[str, Any]:
+    def _get_request_options(
+        self, option_type: RequestOptionType, next_page_token: Optional[Mapping[str, Any]]
+    ) -> MutableMapping[str, Any]:
         options = {}
 
+        token = next_page_token.get("next_page_token") if next_page_token else None
         if (
             self.page_token_option
-            and self._token is not None
+            and token is not None
             and isinstance(self.page_token_option, RequestOption)
             and self.page_token_option.inject_into == option_type
         ):
-            options[self.page_token_option.field_name.eval(config=self.config)] = self._token  # type: ignore # field_name is always cast to an interpolated string
+            options[self.page_token_option.field_name.eval(config=self.config)] = token  # type: ignore # field_name is always cast to an interpolated string
         if (
             self.page_size_option
             and self.pagination_strategy.get_page_size()
@@ -204,6 +212,9 @@ class PaginatorTestReadDecorator(Paginator):
     """
     In some cases, we want to limit the number of requests that are made to the backend source. This class allows for limiting the number of
     pages that are queried throughout a read command.
+
+    WARNING: This decorator is not currently thread-safe like the rest of the low-code framework because it has
+    an internal state to track the current number of pages counted so that it can exit early during a test read
     """
 
     _PAGE_COUNT_BEFORE_FIRST_NEXT_CALL = 1
@@ -217,17 +228,27 @@ class PaginatorTestReadDecorator(Paginator):
         self._decorated = decorated
         self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL
 
+    def get_initial_token(self) -> Optional[Any]:
+        self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL
+        return self._decorated.get_initial_token()
+
     def next_page_token(
-        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any] = None,
     ) -> Optional[Mapping[str, Any]]:
         if self._page_count >= self._maximum_number_of_pages:
             return None
 
         self._page_count += 1
-        return self._decorated.next_page_token(response, last_page_size, last_record)
+        return self._decorated.next_page_token(
+            response, last_page_size, last_record, last_page_token_value
+        )
 
-    def path(self) -> Optional[str]:
-        return self._decorated.path()
+    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+        return self._decorated.path(next_page_token)
 
     def get_request_params(
         self,
@@ -272,7 +293,3 @@ class PaginatorTestReadDecorator(Paginator):
         return self._decorated.get_request_body_json(
             stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token
         )
-
-    def reset(self, reset_value: Optional[Any] = None) -> None:
-        self._decorated.reset()
-        self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL

airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py

@@ -19,7 +19,7 @@ class NoPagination(Paginator):
 
     parameters: InitVar[Mapping[str, Any]]
 
-    def path(self) -> Optional[str]:
+    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
         return None
 
     def get_request_params(
@@ -58,11 +58,14 @@ class NoPagination(Paginator):
     ) -> Mapping[str, Any]:
         return {}
 
+    def get_initial_token(self) -> Optional[Any]:
+        return None
+
     def next_page_token(
-        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
-    ) -> Mapping[str, Any]:
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any],
+    ) -> Optional[Mapping[str, Any]]:
         return {}
-
-    def reset(self, reset_value: Optional[Any] = None) -> None:
-        # No state to reset
-        pass

airbyte_cdk/sources/declarative/requesters/paginators/paginator.py

@@ -24,14 +24,18 @@ class Paginator(ABC, RequestOptionsProvider):
     """
 
     @abstractmethod
-    def reset(self, reset_value: Optional[Any] = None) -> None:
+    def get_initial_token(self) -> Optional[Any]:
         """
-        Reset the pagination's inner state
+        Get the page token that should be included in the request to get the first page of records
         """
 
     @abstractmethod
     def next_page_token(
-        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any],
     ) -> Optional[Mapping[str, Any]]:
         """
         Returns the next_page_token to use to fetch the next page of records.
@@ -39,12 +43,13 @@ class Paginator(ABC, RequestOptionsProvider):
         :param response: the response to process
         :param last_page_size: the number of records read from the response
         :param last_record: the last record extracted from the response
+        :param last_page_token_value: The current value of the page token made on the last request
         :return: A mapping {"next_page_token": <token>} for the next page from the input response object. Returning None means there are no more pages to read in this response.
         """
         pass
 
     @abstractmethod
-    def path(self) -> Optional[str]:
+    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
         """
         Returns the URL path to hit to fetch the next page of records
 

airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py

@@ -43,7 +43,6 @@ class CursorPaginationStrategy(PaginationStrategy):
     )
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        self._initial_cursor = None
         if isinstance(self.cursor_value, str):
             self._cursor_value = InterpolatedString.create(self.cursor_value, parameters=parameters)
         else:
@@ -57,10 +56,19 @@ class CursorPaginationStrategy(PaginationStrategy):
 
     @property
     def initial_token(self) -> Optional[Any]:
-        return self._initial_cursor
+        """
+        CursorPaginationStrategy does not have an initial value because the next cursor is typically included
+        in the response of the first request. For Resumable Full Refresh streams that checkpoint the page
+        cursor, the next cursor should be read from the state or stream slice object.
+        """
+        return None
 
     def next_page_token(
-        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any] = None,
     ) -> Optional[Any]:
         decoded_response = next(self.decoder.decode(response))
 
@@ -87,8 +95,5 @@ class CursorPaginationStrategy(PaginationStrategy):
         )
         return token if token else None
 
-    def reset(self, reset_value: Optional[Any] = None) -> None:
-        self._initial_cursor = reset_value
-
     def get_page_size(self) -> Optional[int]:
         return self.page_size

airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py

@@ -52,7 +52,6 @@ class OffsetIncrement(PaginationStrategy):
     inject_on_first_request: bool = False
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        self._offset = 0
         page_size = str(self.page_size) if isinstance(self.page_size, int) else self.page_size
         if page_size:
             self._page_size: Optional[InterpolatedString] = InterpolatedString(
@@ -64,11 +63,15 @@ class OffsetIncrement(PaginationStrategy):
     @property
     def initial_token(self) -> Optional[Any]:
         if self.inject_on_first_request:
-            return self._offset
+            return 0
         return None
 
     def next_page_token(
-        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any] = None,
     ) -> Optional[Any]:
         decoded_response = next(self.decoder.decode(response))
 
@@ -78,17 +81,16 @@ class OffsetIncrement(PaginationStrategy):
             and last_page_size < self._page_size.eval(self.config, response=decoded_response)
         ) or last_page_size == 0:
             return None
-        else:
-            self._offset += last_page_size
-            return self._offset
-
-    def reset(self, reset_value: Optional[Any] = 0) -> None:
-        if not isinstance(reset_value, int):
+        elif last_page_token_value is None:
+            # If the OffsetIncrement strategy does not inject on the first request, the incoming last_page_token_value
+            # will be None. For this case, we assume that None was the first page and progress to the next offset
+            return 0 + last_page_size
+        elif not isinstance(last_page_token_value, int):
             raise ValueError(
-                f"Reset value {reset_value} for OffsetIncrement pagination strategy was not an integer"
+                f"Last page token value {last_page_token_value} for OffsetIncrement pagination strategy was not an integer"
             )
         else:
-            self._offset = reset_value
+            return last_page_token_value + last_page_size
 
     def get_page_size(self) -> Optional[int]:
         if self._page_size:

airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py

@@ -31,7 +31,6 @@ class PageIncrement(PaginationStrategy):
     inject_on_first_request: bool = False
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        self._page = self.start_from_page
         if isinstance(self.page_size, int) or (self.page_size is None):
             self._page_size = self.page_size
         else:
@@ -43,28 +42,30 @@ class PageIncrement(PaginationStrategy):
     @property
     def initial_token(self) -> Optional[Any]:
         if self.inject_on_first_request:
-            return self._page
+            return self.start_from_page
         return None
 
     def next_page_token(
-        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any],
     ) -> Optional[Any]:
         # Stop paginating when there are fewer records than the page size or the current page has no records
         if (self._page_size and last_page_size < self._page_size) or last_page_size == 0:
             return None
-        else:
-            self._page += 1
-            return self._page
-
-    def reset(self, reset_value: Optional[Any] = None) -> None:
-        if reset_value is None:
-            self._page = self.start_from_page
-        elif not isinstance(reset_value, int):
+        elif last_page_token_value is None:
+            # If the PageIncrement strategy does not inject on the first request, the incoming last_page_token_value
+            # may be None. When this is the case, we assume we've already requested the first page specified by
+            # start_from_page and must now get the next page
+            return self.start_from_page + 1
+        elif not isinstance(last_page_token_value, int):
             raise ValueError(
-                f"Reset value {reset_value} for PageIncrement pagination strategy was not an integer"
+                f"Last page token value {last_page_token_value} for PageIncrement pagination strategy was not an integer"
             )
         else:
-            self._page = reset_value
+            return last_page_token_value + 1
 
     def get_page_size(self) -> Optional[int]:
         return self._page_size

airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py

@@ -4,7 +4,7 @@
 
 from abc import abstractmethod
 from dataclasses import dataclass
-from typing import Any, Optional
+from typing import Any, Mapping, Optional
 
 import requests
 
@@ -26,22 +26,21 @@ class PaginationStrategy:
 
     @abstractmethod
     def next_page_token(
-        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any],
     ) -> Optional[Any]:
         """
         :param response: response to process
         :param last_page_size: the number of records read from the response
         :param last_record: the last record extracted from the response
+        :param last_page_token_value: The current value of the page token made on the last request
         :return: next page token. Returns None if there are no more pages to fetch
         """
         pass
 
-    @abstractmethod
-    def reset(self, reset_value: Optional[Any] = None) -> None:
-        """
-        Reset the pagination's inner state
-        """
-
     @abstractmethod
     def get_page_size(self) -> Optional[int]:
         """

airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py

@@ -44,16 +44,19 @@ class StopConditionPaginationStrategyDecorator(PaginationStrategy):
         self._stop_condition = stop_condition
 
     def next_page_token(
-        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any] = None,
     ) -> Optional[Any]:
-        # We evaluate in reverse order because the assumption is that most of the APIs using data feed structure will return records in
-        # descending order. In terms of performance/memory, we return the records lazily
+        # We evaluate in reverse order because the assumption is that most of the APIs using data feed structure
+        # will return records in descending order. In terms of performance/memory, we return the records lazily
         if last_record and self._stop_condition.is_met(last_record):
             return None
-        return self._delegate.next_page_token(response, last_page_size, last_record)
-
-    def reset(self, reset_value: Optional[Any] = None) -> None:
-        self._delegate.reset(reset_value)
+        return self._delegate.next_page_token(
+            response, last_page_size, last_record, last_page_token_value
+        )
 
     def get_page_size(self) -> Optional[int]:
         return self._delegate.get_page_size()

airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -6,18 +6,7 @@ import json
 from dataclasses import InitVar, dataclass, field
 from functools import partial
 from itertools import islice
-from typing import (
-    Any,
-    Callable,
-    Iterable,
-    List,
-    Mapping,
-    MutableMapping,
-    Optional,
-    Set,
-    Tuple,
-    Union,
-)
+from typing import Any, Callable, Iterable, List, Mapping, Optional, Set, Tuple, Union
 
 import requests
 
@@ -90,9 +79,6 @@ class SimpleRetriever(Retriever):
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._paginator = self.paginator or NoPagination(parameters=parameters)
-        self._last_response: Optional[requests.Response] = None
-        self._last_page_size: int = 0
-        self._last_record: Optional[Record] = None
         self._parameters = parameters
         self._name = (
             InterpolatedString(self._name, parameters=parameters)
@@ -100,10 +86,6 @@ class SimpleRetriever(Retriever):
             else self._name
         )
 
-        # This mapping is used during a resumable full refresh syncs to indicate whether a partition has started syncing
-        # records. Partitions serve as the key and map to True if they already began processing records
-        self._partition_started: MutableMapping[Any, bool] = dict()
-
     @property  # type: ignore
     def name(self) -> str:
         """
@@ -251,17 +233,13 @@ class SimpleRetriever(Retriever):
             raise ValueError("Request body json cannot be a string")
         return body_json
 
-    def _paginator_path(
-        self,
-    ) -> Optional[str]:
+    def _paginator_path(self, next_page_token: Optional[Mapping[str, Any]] = None) -> Optional[str]:
         """
         If the paginator points to a path, follow it, else return nothing so the requester is used.
-        :param stream_state:
-        :param stream_slice:
         :param next_page_token:
         :return:
         """
-        return self._paginator.path()
+        return self._paginator.path(next_page_token=next_page_token)
 
     def _parse_response(
         self,
@@ -272,22 +250,15 @@ class SimpleRetriever(Retriever):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[Record]:
         if not response:
-            self._last_response = None
             yield from []
         else:
-            self._last_response = response
-            record_generator = self.record_selector.select_records(
+            yield from self.record_selector.select_records(
                 response=response,
                 stream_state=stream_state,
                 records_schema=records_schema,
                 stream_slice=stream_slice,
                 next_page_token=next_page_token,
             )
-            self._last_page_size = 0
-            for record in record_generator:
-                self._last_page_size += 1
-                self._last_record = record
-                yield record
 
     @property  # type: ignore
     def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]:
@@ -299,7 +270,13 @@ class SimpleRetriever(Retriever):
         if not isinstance(value, property):
             self._primary_key = value
 
-    def _next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+    def _next_page_token(
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any],
+    ) -> Optional[Mapping[str, Any]]:
         """
         Specifies a pagination strategy.
 
@@ -307,7 +284,12 @@ class SimpleRetriever(Retriever):
 
         :return: The token for the next page from the input response object. Returning None means there are no more pages to read in this response.
         """
-        return self._paginator.next_page_token(response, self._last_page_size, self._last_record)
+        return self._paginator.next_page_token(
+            response=response,
+            last_page_size=last_page_size,
+            last_record=last_record,
+            last_page_token_value=last_page_token_value,
+        )
 
     def _fetch_next_page(
         self,
@@ -316,7 +298,7 @@ class SimpleRetriever(Retriever):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Optional[requests.Response]:
         return self.requester.send_request(
-            path=self._paginator_path(),
+            path=self._paginator_path(next_page_token=next_page_token),
             stream_state=stream_state,
             stream_slice=stream_slice,
             next_page_token=next_page_token,
@@ -345,20 +327,37 @@ class SimpleRetriever(Retriever):
     # This logic is similar to _read_pages in the HttpStream class. When making changes here, consider making changes there as well.
     def _read_pages(
         self,
-        records_generator_fn: Callable[[Optional[requests.Response]], Iterable[StreamData]],
+        records_generator_fn: Callable[[Optional[requests.Response]], Iterable[Record]],
         stream_state: Mapping[str, Any],
         stream_slice: StreamSlice,
-    ) -> Iterable[StreamData]:
+    ) -> Iterable[Record]:
         pagination_complete = False
-        next_page_token = None
+        initial_token = self._paginator.get_initial_token()
+        next_page_token: Optional[Mapping[str, Any]] = (
+            {"next_page_token": initial_token} if initial_token else None
+        )
         while not pagination_complete:
             response = self._fetch_next_page(stream_state, stream_slice, next_page_token)
-            yield from records_generator_fn(response)
+
+            last_page_size = 0
+            last_record: Optional[Record] = None
+            for record in records_generator_fn(response):
+                last_page_size += 1
+                last_record = record
+                yield record
 
             if not response:
                 pagination_complete = True
             else:
-                next_page_token = self._next_page_token(response)
+                last_page_token_value = (
+                    next_page_token.get("next_page_token") if next_page_token else None
+                )
+                next_page_token = self._next_page_token(
+                    response=response,
+                    last_page_size=last_page_size,
+                    last_record=last_record,
+                    last_page_token_value=last_page_token_value,
+                )
                 if not next_page_token:
                     pagination_complete = True
 
@@ -367,19 +366,38 @@ class SimpleRetriever(Retriever):
 
     def _read_single_page(
         self,
-        records_generator_fn: Callable[[Optional[requests.Response]], Iterable[StreamData]],
+        records_generator_fn: Callable[[Optional[requests.Response]], Iterable[Record]],
         stream_state: Mapping[str, Any],
         stream_slice: StreamSlice,
     ) -> Iterable[StreamData]:
-        response = self._fetch_next_page(stream_state, stream_slice)
-        yield from records_generator_fn(response)
+        initial_token = stream_state.get("next_page_token")
+        if initial_token is None:
+            initial_token = self._paginator.get_initial_token()
+        next_page_token: Optional[Mapping[str, Any]] = (
+            {"next_page_token": initial_token} if initial_token else None
+        )
+
+        response = self._fetch_next_page(stream_state, stream_slice, next_page_token)
+
+        last_page_size = 0
+        last_record: Optional[Record] = None
+        for record in records_generator_fn(response):
+            last_page_size += 1
+            last_record = record
+            yield record
 
         if not response:
-            next_page_token: Mapping[str, Any] = {FULL_REFRESH_SYNC_COMPLETE_KEY: True}
+            next_page_token = {FULL_REFRESH_SYNC_COMPLETE_KEY: True}
         else:
-            next_page_token = self._next_page_token(response) or {
-                FULL_REFRESH_SYNC_COMPLETE_KEY: True
-            }
+            last_page_token_value = (
+                next_page_token.get("next_page_token") if next_page_token else None
+            )
+            next_page_token = self._next_page_token(
+                response=response,
+                last_page_size=last_page_size,
+                last_record=last_record,
+                last_page_token_value=last_page_token_value,
+            ) or {FULL_REFRESH_SYNC_COMPLETE_KEY: True}
 
         if self.cursor:
             self.cursor.close_slice(
@@ -414,25 +432,14 @@ class SimpleRetriever(Retriever):
         if self.cursor and isinstance(self.cursor, ResumableFullRefreshCursor):
             stream_state = self.state
 
-            # Before syncing the RFR stream, we check if the job's prior attempt was successful and don't need to fetch more records
-            # The platform deletes stream state for full refresh streams before starting a new job, so we don't need to worry about
-            # this value existing for the initial attempt
+            # Before syncing the RFR stream, we check if the job's prior attempt was successful and don't need to
+            # fetch more records. The platform deletes stream state for full refresh streams before starting a
+            # new job, so we don't need to worry about this value existing for the initial attempt
             if stream_state.get(FULL_REFRESH_SYNC_COMPLETE_KEY):
                 return
-            cursor_value = stream_state.get("next_page_token")
-
-            # The first attempt to read a page for the current partition should reset the paginator to the current
-            # cursor state which is initially assigned to the incoming state from the platform
-            partition_key = self._to_partition_key(_slice.partition)
-            if partition_key not in self._partition_started:
-                self._partition_started[partition_key] = True
-                self._paginator.reset(reset_value=cursor_value)
 
             yield from self._read_single_page(record_generator, stream_state, _slice)
         else:
-            # Fixing paginator types has a long tail of dependencies
-            self._paginator.reset()
-
             for stream_data in self._read_pages(record_generator, self.state, _slice):
                 current_record = self._extract_record(stream_data, _slice)
                 if self.cursor and current_record:
@@ -518,7 +525,7 @@ class SimpleRetriever(Retriever):
         stream_state: Mapping[str, Any],
         records_schema: Mapping[str, Any],
         stream_slice: Optional[StreamSlice],
-    ) -> Iterable[StreamData]:
+    ) -> Iterable[Record]:
         yield from self._parse_response(
             response,
             stream_slice=stream_slice,
@@ -562,7 +569,7 @@ class SimpleRetrieverTestReadDecorator(SimpleRetriever):
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Optional[requests.Response]:
         return self.requester.send_request(
-            path=self._paginator_path(),
+            path=self._paginator_path(next_page_token=next_page_token),
             stream_state=stream_state,
             stream_slice=stream_slice,
             next_page_token=next_page_token,

airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -16,7 +16,7 @@ class DeclarativePartitionFactory:
         self,
         stream_name: str,
         json_schema: Mapping[str, Any],
-        retriever_factory: Callable[[], Retriever],
+        retriever: Retriever,
         message_repository: MessageRepository,
     ) -> None:
         """
@@ -26,14 +26,14 @@ class DeclarativePartitionFactory:
         """
         self._stream_name = stream_name
         self._json_schema = json_schema
-        self._retriever_factory = retriever_factory
+        self._retriever = retriever
         self._message_repository = message_repository
 
     def create(self, stream_slice: StreamSlice) -> Partition:
         return DeclarativePartition(
             self._stream_name,
             self._json_schema,
-            self._retriever_factory(),
+            self._retriever,
             self._message_repository,
             stream_slice,
         )

airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py

@@ -31,12 +31,6 @@ class DeliverRawFiles(BaseModel):
 
     delivery_type: Literal["use_file_transfer"] = Field("use_file_transfer", const=True)
 
-    preserve_subdirectories_directories: bool = Field(
-        title="Preserve Subdirectories in File Paths",
-        description="If enabled replicate source folder structure",
-        default=True,
-    )
-
 
 class AbstractFileBasedSpec(BaseModel):
     """

airbyte_cdk/sources/file_based/exceptions.py

@@ -111,10 +111,6 @@ class ErrorListingFiles(BaseFileBasedSourceError):
     pass
 
 
-class DuplicatedFilesError(BaseFileBasedSourceError):
-    pass
-
-
 class CustomFileBasedException(AirbyteTracedException):
     """
     A specialized exception for file-based connectors.
@@ -127,25 +123,3 @@ class CustomFileBasedException(AirbyteTracedException):
 
 class FileSizeLimitError(CustomFileBasedException):
     pass
-
-
-def format_duplicate_files_error_message(
-    stream_name: str, duplicated_files_names: List[dict[str, List[str]]]
-) -> str:
-    duplicated_files_messages = []
-    for duplicated_file in duplicated_files_names:
-        for duplicated_file_name, file_paths in duplicated_file.items():
-            file_duplicated_message = (
-                f"{len(file_paths)} duplicates found for file name {duplicated_file_name}:\n\n"
-                + "".join(f"\n - {file_paths}")
-            )
-            duplicated_files_messages.append(file_duplicated_message)
-
-    error_message = (
-        f"ERROR: Duplicate filenames found for stream {stream_name}. "
-        "Duplicate file names are not allowed if the Preserve Subdirectories in File Paths option is disabled. "
-        "Please remove or rename the duplicate files before attempting to re-run the sync.\n\n"
-        + "\n".join(duplicated_files_messages)
-    )
-
-    return error_message

airbyte_cdk/sources/file_based/file_based_source.py

@@ -242,7 +242,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         stream=self._make_default_stream(
                             stream_config=stream_config,
                             cursor=cursor,
-                            parsed_config=parsed_config,
+                            use_file_transfer=self._use_file_transfer(parsed_config),
                         ),
                         source=self,
                         logger=self.logger,
@@ -273,7 +273,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                         stream=self._make_default_stream(
                             stream_config=stream_config,
                             cursor=cursor,
-                            parsed_config=parsed_config,
+                            use_file_transfer=self._use_file_transfer(parsed_config),
                         ),
                         source=self,
                         logger=self.logger,
@@ -285,7 +285,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                     stream = self._make_default_stream(
                         stream_config=stream_config,
                         cursor=cursor,
-                        parsed_config=parsed_config,
+                        use_file_transfer=self._use_file_transfer(parsed_config),
                     )
 
                 streams.append(stream)
@@ -298,7 +298,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
         self,
         stream_config: FileBasedStreamConfig,
         cursor: Optional[AbstractFileBasedCursor],
-        parsed_config: AbstractFileBasedSpec,
+        use_file_transfer: bool = False,
     ) -> AbstractFileBasedStream:
         return DefaultFileBasedStream(
             config=stream_config,
@@ -310,10 +310,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             validation_policy=self._validate_and_get_validation_policy(stream_config),
             errors_collector=self.errors_collector,
             cursor=cursor,
-            use_file_transfer=self._use_file_transfer(parsed_config),
-            preserve_subdirectories_directories=self._preserve_subdirectories_directories(
-                parsed_config
-            ),
+            use_file_transfer=use_file_transfer,
         )
 
     def _get_stream_from_catalog(
@@ -388,14 +385,3 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
             and parsed_config.delivery_method.delivery_type == "use_file_transfer"
         )
         return use_file_transfer
-
-    @staticmethod
-    def _preserve_subdirectories_directories(parsed_config: AbstractFileBasedSpec) -> bool:
-        # fall back to preserve subdirectories if config is not present or incomplete
-        if (
-            FileBasedSource._use_file_transfer(parsed_config)
-            and hasattr(parsed_config.delivery_method, "preserve_subdirectories_directories")
-            and parsed_config.delivery_method.preserve_subdirectories_directories is not None
-        ):
-            return parsed_config.delivery_method.preserve_subdirectories_directories
-        return True

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -135,16 +135,6 @@ class AbstractFileBasedStreamReader(ABC):
             return use_file_transfer
         return False
 
-    def preserve_subdirectories_directories(self) -> bool:
-        # fall back to preserve subdirectories if config is not present or incomplete
-        if (
-            self.use_file_transfer()
-            and hasattr(self.config.delivery_method, "preserve_subdirectories_directories")
-            and self.config.delivery_method.preserve_subdirectories_directories is not None
-        ):
-            return self.config.delivery_method.preserve_subdirectories_directories
-        return True
-
     @abstractmethod
     def get_file(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
@@ -169,13 +159,10 @@ class AbstractFileBasedStreamReader(ABC):
         """
         ...
 
-    def _get_file_transfer_paths(self, file: RemoteFile, local_directory: str) -> List[str]:
-        preserve_subdirectories_directories = self.preserve_subdirectories_directories()
-        if preserve_subdirectories_directories:
-            # Remove left slashes from source path format to make relative path for writing locally
-            file_relative_path = file.uri.lstrip("/")
-        else:
-            file_relative_path = path.basename(file.uri)
+    @staticmethod
+    def _get_file_transfer_paths(file: RemoteFile, local_directory: str) -> List[str]:
+        # Remove left slashes from source path format to make relative path for writing locally
+        file_relative_path = file.uri.lstrip("/")
         local_file_path = path.join(local_directory, file_relative_path)
 
         # Ensure the local directory exists

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -5,24 +5,20 @@
 import asyncio
 import itertools
 import traceback
-from collections import defaultdict
 from copy import deepcopy
 from functools import cache
-from os import path
-from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union
+from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Union
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
 from airbyte_cdk.sources.file_based.exceptions import (
-    DuplicatedFilesError,
     FileBasedSourceError,
     InvalidSchemaError,
     MissingSchemaError,
     RecordParseError,
     SchemaInferenceError,
     StopSyncPerValidationPolicy,
-    format_duplicate_files_error_message,
 )
 from airbyte_cdk.sources.file_based.file_types import FileTransfer
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
@@ -47,8 +43,6 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     """
 
     FILE_TRANSFER_KW = "use_file_transfer"
-    PRESERVE_SUBDIRECTORIES_KW = "preserve_subdirectories_directories"
-    FILES_KEY = "files"
     DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
     ab_last_mod_col = "_ab_source_file_last_modified"
     ab_file_name_col = "_ab_source_file_url"
@@ -56,14 +50,10 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     source_file_url = "source_file_url"
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
     use_file_transfer = False
-    preserve_subdirectories_directories = True
 
     def __init__(self, **kwargs: Any):
         if self.FILE_TRANSFER_KW in kwargs:
             self.use_file_transfer = kwargs.pop(self.FILE_TRANSFER_KW, False)
-            self.preserve_subdirectories_directories = kwargs.pop(
-                self.PRESERVE_SUBDIRECTORIES_KW, True
-            )
         super().__init__(**kwargs)
 
     @property
@@ -108,43 +98,15 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
 
-    def _duplicated_files_names(
-        self, slices: List[dict[str, List[RemoteFile]]]
-    ) -> List[dict[str, List[str]]]:
-        seen_file_names = set()
-        duplicates_file_names = set()
-        file_paths = defaultdict(list)
-        for file_slice in slices:
-            for file_found in file_slice[self.FILES_KEY]:
-                file_name = path.basename(file_found.uri)
-                if file_name not in seen_file_names:
-                    seen_file_names.add(file_name)
-                else:
-                    duplicates_file_names.add(file_name)
-                file_paths[file_name].append(file_found.uri)
-        return [
-            {duplicated_file: file_paths[duplicated_file]}
-            for duplicated_file in duplicates_file_names
-        ]
-
     def compute_slices(self) -> Iterable[Optional[Mapping[str, Any]]]:
         # Sort files by last_modified, uri and return them grouped by last_modified
         all_files = self.list_files()
         files_to_read = self._cursor.get_files_to_sync(all_files, self.logger)
         sorted_files_to_read = sorted(files_to_read, key=lambda f: (f.last_modified, f.uri))
         slices = [
-            {self.FILES_KEY: list(group[1])}
+            {"files": list(group[1])}
             for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)
         ]
-        if slices and not self.preserve_subdirectories_directories:
-            duplicated_files_names = self._duplicated_files_names(slices)
-            if duplicated_files_names:
-                raise DuplicatedFilesError(
-                    format_duplicate_files_error_message(
-                        stream_name=self.name, duplicated_files_names=duplicated_files_names
-                    ),
-                    stream=self.name,
-                )
         return slices
 
     def transform_record(

{airbyte_cdk-6.13.1.dev41012.dist-info → airbyte_cdk-6.14.0.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.13.1.dev41012
+Version: 6.14.0.dev1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.13.1.dev41012.dist-info → airbyte_cdk-6.14.0.dev1.dist-info}/RECORD

@@ -62,7 +62,7 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQr
 airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
 airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
 airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
-airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=PxP4p2686wsf1gjsumGKnh2o2Jjnrqg8QLGijEIrp-A,23412
+airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=v61HsAm_TmkhxbvOQS7Qvo4sNou-n9GtUT8thams6i0,22480
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
@@ -99,7 +99,7 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha
 airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=LYEZnZ_hB7rvBSZxG9s0RSrzsOkDWbBY0_P6qu5lEfc,3212
 airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=-V5UddGm69UKEB6o_O1EIES9kfY8FV_X4Ji8w1yOuSA,981
 airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=BtsY_jtT4MihFqeQgc05HXj3Ndt-e2ESQgGwbg3Sdxc,6430
-airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=jf24RK-1fBhTYDpcGEakZtGNNJfG5NS8CCF5bEgNmRo,3977
+airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=Y5AWYxbJTUtJ_Jm7DV9qrZDiymFR9LST7fBt4piT2-U,4585
 airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=wX_dQ401siuwh3zHgSHRnSN1vIojI4Nufg3BwzZAzk0,16239
 airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
@@ -135,15 +135,15 @@ airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.p
 airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=o0520AmHMb7SAoeokVNwoOzuZzIAT6ryx9uFYGSOrs0,8664
 airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=RqYPkgJFAWfcZBTc-JBcGHPm4JL1ZQOhs9GKU4MP2eE,14723
 airbyte_cdk/sources/declarative/requesters/paginators/__init__.py,sha256=uArbKs9JKNCt7t9tZoeWwjDpyI1HoPp29FNW0JzvaEM,644
-airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=LxTq1hieznRWlYlfODdZbMDUml-g6NyBkdwVI2mCNMM,10910
-airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=-P-QOlefFhEe99bsB2y3yTvA8c8kCCbfBaTS6qPvF6I,1927
-airbyte_cdk/sources/declarative/requesters/paginators/paginator.py,sha256=ZgyvH7DOrASQ5K__J5SRAXH3REUW2n3yPHnFW9xq4NU,1972
+airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=FnSl3qPvv5wD6ieAI2Ic5c4dqBk-3fRe4tCaWzq3YwM,11840
+airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=j6j9QRPaTbKQ2N661RFVKthhkWiodEp6ut0tKeEd0Ng,2019
+airbyte_cdk/sources/declarative/requesters/paginators/paginator.py,sha256=OlN-y0PEOMzlUNUh3pzonoTpIJpGwkP4ibFengvpLVU,2230
 airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py,sha256=2gly8fuZpDNwtu1Qg6oE2jBLGqQRdzSLJdnpk_iDV6I,767
-airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py,sha256=vFzpNv8BdgXrYO5qhi2_Un4x4y-EAQWxinZtEPWz5KI,3654
-airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py,sha256=pMPi6iQrhtrI9VRPj218QNM_OqD8lX8P3Tr9yloKoE8,3367
-airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py,sha256=kQGpfr-dOwarxTIf2S4sHVulBzm8zSwQXBM7rOhkafA,2491
-airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py,sha256=ABpO4t0UUziBZnyml8UT_NhlF6loekhQji57TpKnaiY,1290
-airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py,sha256=-8NwokW-aKwv8DdeHh1ssODTobBYSOmIhH2-IjSjlNA,2213
+airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py,sha256=yLzzK5YIRTkXd2Z-BS__AZXuTd6HXjJIxq05K-lQoxI,3898
+airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py,sha256=WvGt_DTFcAgTR-NHrlrR7B71yG-L6jmfW-Gwm9iYzjY,3624
+airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py,sha256=Z2i6a-oKMmOTxHxsTVSnyaShkJ3u8xZw1xIJdx2yxss,2731
+airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py,sha256=UiHQI2lsRDPqM4nMvKMnmsXA3gFg5BFE4lCPEBhuCTs,1317
+airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py,sha256=LoKXdUbSgHEtSwtA8DFrnX6SpQbRVVwreY8NguTKTcI,2229
 airbyte_cdk/sources/declarative/requesters/request_option.py,sha256=_qmv8CLQQ3fERt6BuMZeRu6tZXscPoeARx1VJdWMQ_M,1055
 airbyte_cdk/sources/declarative/requesters/request_options/__init__.py,sha256=WCwpKqM4wKqy-DHJaCHbKAlFqRVOqMi9K5qonxIfi_Y,809
 airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py,sha256=FLkg0uzC9bc-zFnALWr0FLYpKsz8iK2xQsd4UOyeW08,3706
@@ -161,7 +161,7 @@ airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=Aio
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=ix9m1dkR69DcXCXUKC5RK_ZZM7ojTLBQ4IkWQTfmfCk,456
 airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=_-d3MvHh-4r46i4wjQikD4ZygKA7TvuDu2i04qqULEg,3731
 airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
-airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=N4swGw5mfuTXJ2R7AKX18CHzizsr69pXwt5uSHLPi48,24172
+airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=jxQ_9xcVD07r9PKhofitAqMkdX1k8ZNyy50qz5NwkFs,24540
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=HztgVVaZdil5UfgUZcv_Hyy84r89_EKRwyO2hoewNVg,749
 airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=KTACrIE23a83wsm3Rd9Eb4K6-20lrGqYxTHNp9yxsso,1820
 airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=H6A3NQ6kPPM-cUNPmdvDPc9xNzR1rQNrK95GbgCW334,8822
@@ -171,7 +171,7 @@ airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLn
 airbyte_cdk/sources/declarative/spec/__init__.py,sha256=H0UwoRhgucbKBIzg85AXrifybVmfpwWpPdy22vZKVuo,141
 airbyte_cdk/sources/declarative/spec/spec.py,sha256=ODSNUgkDOhnLQnwLjgSaME6R3kNeywjROvbNrWEnsgU,1876
 airbyte_cdk/sources/declarative/stream_slicers/__init__.py,sha256=sI9vhc95RwJYOnA0VKjcbtKgFcmAbWjhdWBXFbAijOs,176
-airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=E7feZ5xkHwFHODq8FSjwdGe291RZoCMCRHT1rWnQ1lI,3463
+airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=ldmfzOhkA8yMPQKDOHO-bO8zUYJ0oVAs8BIZ-O57exk,3415
 airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
 airbyte_cdk/sources/declarative/transformations/__init__.py,sha256=CPJ8TlMpiUmvG3624VYu_NfTzxwKcfBjM2Q2wJ7fkSA,919
 airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=r4YdAuAk2bQtNWJMztIIy2CC-NglD9NeK1s1TeO9wkw,5027
@@ -193,7 +193,7 @@ airbyte_cdk/sources/file_based/availability_strategy/__init__.py,sha256=ddKQfUmk
 airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py,sha256=01Nd4b7ERAbp-OZo_8rrAzFXWPTMwr02SnWiN17nx8Q,2363
 airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py,sha256=j9T5TimfWFUz7nqsaj-83G3xWmDpsmeSbDnaUNmz0UM,5849
 airbyte_cdk/sources/file_based/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py,sha256=gigLToFfWcRhE2ppPFyojW2S1boMc5dtrO4w7BZjaks,6662
+airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py,sha256=tj-M1L5BTa5yIQ3jHo09CtCTSq_eR-68zgyOPqwsurw,6455
 airbyte_cdk/sources/file_based/config/avro_format.py,sha256=NxTF96ewzn6HuhgodsY7Rpb-ybr1ZEWW5d4Vid64g5A,716
 airbyte_cdk/sources/file_based/config/csv_format.py,sha256=NWekkyT8dTwiVK0mwa_krQD4FJPHSDfILo8kPAg3-Vs,8006
 airbyte_cdk/sources/file_based/config/excel_format.py,sha256=9qAmTsT6SoVzNfNv0oBVkVCmiyqQuVAbfRKajjoa7Js,378
@@ -204,9 +204,9 @@ airbyte_cdk/sources/file_based/config/unstructured_format.py,sha256=tIbB9Pn1HqU6
 airbyte_cdk/sources/file_based/discovery_policy/__init__.py,sha256=gl3ey6mZbyfraB9P3pFhf9UJp2JeTZ1SUFAopy2iBvY,301
 airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py,sha256=dCfXX529Rd5rtopg4VeEgTPJjFtqjtjzPq6LCw18Wt0,605
 airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py,sha256=-xujTidtrq6HC00WKbjQh1CZdT5LMuzkp5BLjqDmfTY,1007
-airbyte_cdk/sources/file_based/exceptions.py,sha256=KfOgQgssBKgsv3h5po2IG1DhZcH664Zf_fx96mBlPSg,6761
-airbyte_cdk/sources/file_based/file_based_source.py,sha256=0rKW1XQjj6Up-e_y5ls5b9k7dWAASQHkuWy7RaDxs9Y,17303
-airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=99CiCmsTj9LzO0QR90lB0vKTBR5TTCAyDCO8PURCris,6877
+airbyte_cdk/sources/file_based/exceptions.py,sha256=AEELNIRzKPX6eopKd_2jhE7WiNeR0Aw7nQWVOL8fvkc,5760
+airbyte_cdk/sources/file_based/file_based_source.py,sha256=RfpctRNLJ_EHKKEc2E1EZGYRfhG0Z9o6TgsKS4XrSNY,16652
+airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=ohxKlqPuV7TGwjyRy_gaWUol8QN5lBSoCYoaqBtRh1c,6179
 airbyte_cdk/sources/file_based/file_types/__init__.py,sha256=blCLn0-2LC-ZdgcNyDEhqM2RiUvEjEBh-G4-t32ZtuM,1268
 airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=XNx-JC-sgzH9u3nOJ2M59FxBXvtig8LN6BIkeDOavZA,10858
 airbyte_cdk/sources/file_based/file_types/csv_parser.py,sha256=QlCXB-ry3np67Q_VerQEPoWDOTcPTB6Go4ydZxY9ae4,20445
@@ -232,7 +232,7 @@ airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_c
 airbyte_cdk/sources/file_based/stream/cursor/__init__.py,sha256=MhFB5hOo8sjwvCh8gangaymdg3EJWYt_72brFOZt068,191
 airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py,sha256=om-x3gZFPgWDpi15S9RxZmR36VHnk8sytgN6LlBQhAw,1934
 airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py,sha256=VGV7xLyBribuBMVrXtO1xqkWJD86bl7yhXtjnwLMohM,7051
-airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=HRjR0rQGc8cYK2PxpLgAvZQ--jvtV8QgS1QIxkemnko,18413
+airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=rpwU6AOyhFLuXtcFKkcOHFWbRQ4kLCOKzAjcID_M87k,16770
 airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
 airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
 airbyte_cdk/sources/http_logger.py,sha256=TyBmtRA6D9g0XDkKGvdM415b36RXDjgfkwRewDsH8-0,1576
@@ -340,8 +340,8 @@ airbyte_cdk/utils/slice_hasher.py,sha256=-pHexlNYoWYPnXNH-M7HEbjmeJe9Zk7SJijdQ7d
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.13.1.dev41012.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.13.1.dev41012.dist-info/METADATA,sha256=HsxDkbakEzPNjuI4b9lbu35vVlb-tvTb2rwv9yddG88,5997
-airbyte_cdk-6.13.1.dev41012.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-airbyte_cdk-6.13.1.dev41012.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.13.1.dev41012.dist-info/RECORD,,
+airbyte_cdk-6.14.0.dev1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.14.0.dev1.dist-info/METADATA,sha256=IE3J33y4yRYF6vsHR7l-BFHI4_K1LVq5S431ivv5Sos,5993
+airbyte_cdk-6.14.0.dev1.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+airbyte_cdk-6.14.0.dev1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.14.0.dev1.dist-info/RECORD,,