airbyte-cdk 6.13.0.dev0__tar.gz → 6.13.1.dev4101__tar.gz

{airbyte_cdk-6.13.0.dev0 → airbyte_cdk-6.13.1.dev4101}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.13.0.dev0
+Version: 6.13.1.dev4101
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.13.0.dev0 → airbyte_cdk-6.13.1.dev4101}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -194,11 +194,10 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             # Some low-code sources use a combination of DeclarativeStream and regular Python streams. We can't inspect
             # these legacy Python streams the way we do low-code streams to determine if they are concurrent compatible,
             # so we need to treat them as synchronous
-            if isinstance(declarative_stream, DeclarativeStream) and (
-                name_to_stream_mapping[declarative_stream.name]["retriever"]["type"]
+            if (
+                isinstance(declarative_stream, DeclarativeStream)
+                and name_to_stream_mapping[declarative_stream.name]["retriever"]["type"]
                 == "SimpleRetriever"
-                or name_to_stream_mapping[declarative_stream.name]["retriever"]["type"]
-                == "AsyncRetriever"
             ):
                 incremental_sync_component_definition = name_to_stream_mapping[
                     declarative_stream.name
@@ -218,11 +217,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                     and not incremental_sync_component_definition
                 )
 
-                is_async_job_stream = (
-                    name_to_stream_mapping[declarative_stream.name].get("retriever", {}).get("type")
-                    == "AsyncRetriever"
-                )
-
                 if self._is_datetime_incremental_without_partition_routing(
                     declarative_stream, incremental_sync_component_definition
                 ):
@@ -274,26 +268,15 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                 elif (
                     is_substream_without_incremental or is_without_partition_router_or_cursor
                 ) and hasattr(declarative_stream.retriever, "stream_slicer"):
-                    if is_async_job_stream:
-                        # A stream's AsyncRetriever must be shared across all partitions because it uses a
-                        # shared JobRepository to manage the state of jobs requests and when they are ready
-                        async_retriever = declarative_stream.retriever
-
-                        def async_retriever_factory_method() -> Retriever:
-                            return async_retriever
-
-                        retriever_factory = async_retriever_factory_method
-                    else:
-                        retriever_factory = self._retriever_factory(
-                            name_to_stream_mapping[declarative_stream.name],
-                            config,
-                            {},
-                        )
                     partition_generator = StreamSlicerPartitionGenerator(
                         DeclarativePartitionFactory(
                             declarative_stream.name,
                             declarative_stream.get_json_schema(),
-                            retriever_factory,
+                            self._retriever_factory(
+                                name_to_stream_mapping[declarative_stream.name],
+                                config,
+                                {},
+                            ),
                             self.message_repository,
                         ),
                         declarative_stream.retriever.stream_slicer,

{airbyte_cdk-6.13.0.dev0 → airbyte_cdk-6.13.1.dev4101}/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py

@@ -112,39 +112,27 @@ class DefaultPaginator(Paginator):
             )
         if isinstance(self.url_base, str):
             self.url_base = InterpolatedString(string=self.url_base, parameters=parameters)
-
-    def get_initial_token(self) -> Optional[Any]:
-        """
-        Return the page token that should be used for the first request of a stream
-
-        WARNING: get_initial_token() should not be used by streams that use RFR that perform checkpointing
-        of state using page numbers. Because paginators are stateless
-        """
-        return self.pagination_strategy.initial_token
+        self._token: Optional[Any] = self.pagination_strategy.initial_token
 
     def next_page_token(
-        self,
-        response: requests.Response,
-        last_page_size: int,
-        last_record: Optional[Record],
-        last_page_token_value: Optional[Any] = None,
+        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
     ) -> Optional[Mapping[str, Any]]:
-        next_page_token = self.pagination_strategy.next_page_token(
-            response=response,
-            last_page_size=last_page_size,
-            last_record=last_record,
-            last_page_token_value=last_page_token_value,
+        self._token = self.pagination_strategy.next_page_token(
+            response, last_page_size, last_record
         )
-        if next_page_token:
-            return {"next_page_token": next_page_token}
+        if self._token:
+            return {"next_page_token": self._token}
         else:
             return None
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
-        token = next_page_token.get("next_page_token") if next_page_token else None
-        if token and self.page_token_option and isinstance(self.page_token_option, RequestPath):
+    def path(self) -> Optional[str]:
+        if (
+            self._token
+            and self.page_token_option
+            and isinstance(self.page_token_option, RequestPath)
+        ):
             # Replace url base to only return the path
-            return str(token).replace(self.url_base.eval(self.config), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
+            return str(self._token).replace(self.url_base.eval(self.config), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
         else:
             return None
 
@@ -155,7 +143,7 @@ class DefaultPaginator(Paginator):
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> MutableMapping[str, Any]:
-        return self._get_request_options(RequestOptionType.request_parameter, next_page_token)
+        return self._get_request_options(RequestOptionType.request_parameter)
 
     def get_request_headers(
         self,
@@ -164,7 +152,7 @@ class DefaultPaginator(Paginator):
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, str]:
-        return self._get_request_options(RequestOptionType.header, next_page_token)
+        return self._get_request_options(RequestOptionType.header)
 
     def get_request_body_data(
         self,
@@ -173,7 +161,7 @@ class DefaultPaginator(Paginator):
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, Any]:
-        return self._get_request_options(RequestOptionType.body_data, next_page_token)
+        return self._get_request_options(RequestOptionType.body_data)
 
     def get_request_body_json(
         self,
@@ -182,21 +170,25 @@ class DefaultPaginator(Paginator):
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Mapping[str, Any]:
-        return self._get_request_options(RequestOptionType.body_json, next_page_token)
+        return self._get_request_options(RequestOptionType.body_json)
 
-    def _get_request_options(
-        self, option_type: RequestOptionType, next_page_token: Optional[Mapping[str, Any]]
-    ) -> MutableMapping[str, Any]:
+    def reset(self, reset_value: Optional[Any] = None) -> None:
+        if reset_value:
+            self.pagination_strategy.reset(reset_value=reset_value)
+        else:
+            self.pagination_strategy.reset()
+        self._token = self.pagination_strategy.initial_token
+
+    def _get_request_options(self, option_type: RequestOptionType) -> MutableMapping[str, Any]:
         options = {}
 
-        token = next_page_token.get("next_page_token") if next_page_token else None
         if (
             self.page_token_option
-            and token is not None
+            and self._token is not None
             and isinstance(self.page_token_option, RequestOption)
             and self.page_token_option.inject_into == option_type
         ):
-            options[self.page_token_option.field_name.eval(config=self.config)] = token  # type: ignore # field_name is always cast to an interpolated string
+            options[self.page_token_option.field_name.eval(config=self.config)] = self._token  # type: ignore # field_name is always cast to an interpolated string
         if (
             self.page_size_option
             and self.pagination_strategy.get_page_size()
@@ -212,9 +204,6 @@ class PaginatorTestReadDecorator(Paginator):
     """
     In some cases, we want to limit the number of requests that are made to the backend source. This class allows for limiting the number of
     pages that are queried throughout a read command.
-
-    WARNING: This decorator is not currently thread-safe like the rest of the low-code framework because it has
-    an internal state to track the current number of pages counted so that it can exit early during a test read
     """
 
     _PAGE_COUNT_BEFORE_FIRST_NEXT_CALL = 1
@@ -228,27 +217,17 @@ class PaginatorTestReadDecorator(Paginator):
         self._decorated = decorated
         self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL
 
-    def get_initial_token(self) -> Optional[Any]:
-        self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL
-        return self._decorated.get_initial_token()
-
     def next_page_token(
-        self,
-        response: requests.Response,
-        last_page_size: int,
-        last_record: Optional[Record],
-        last_page_token_value: Optional[Any] = None,
+        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
     ) -> Optional[Mapping[str, Any]]:
         if self._page_count >= self._maximum_number_of_pages:
             return None
 
         self._page_count += 1
-        return self._decorated.next_page_token(
-            response, last_page_size, last_record, last_page_token_value
-        )
+        return self._decorated.next_page_token(response, last_page_size, last_record)
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
-        return self._decorated.path(next_page_token)
+    def path(self) -> Optional[str]:
+        return self._decorated.path()
 
     def get_request_params(
         self,
@@ -293,3 +272,7 @@ class PaginatorTestReadDecorator(Paginator):
         return self._decorated.get_request_body_json(
             stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token
         )
+
+    def reset(self, reset_value: Optional[Any] = None) -> None:
+        self._decorated.reset()
+        self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL

{airbyte_cdk-6.13.0.dev0 → airbyte_cdk-6.13.1.dev4101}/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py

@@ -19,7 +19,7 @@ class NoPagination(Paginator):
 
     parameters: InitVar[Mapping[str, Any]]
 
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+    def path(self) -> Optional[str]:
         return None
 
     def get_request_params(
@@ -58,14 +58,11 @@ class NoPagination(Paginator):
     ) -> Mapping[str, Any]:
         return {}
 
-    def get_initial_token(self) -> Optional[Any]:
-        return None
-
     def next_page_token(
-        self,
-        response: requests.Response,
-        last_page_size: int,
-        last_record: Optional[Record],
-        last_page_token_value: Optional[Any],
-    ) -> Optional[Mapping[str, Any]]:
+        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
+    ) -> Mapping[str, Any]:
         return {}
+
+    def reset(self, reset_value: Optional[Any] = None) -> None:
+        # No state to reset
+        pass

{airbyte_cdk-6.13.0.dev0 → airbyte_cdk-6.13.1.dev4101}/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py

@@ -24,18 +24,14 @@ class Paginator(ABC, RequestOptionsProvider):
     """
 
     @abstractmethod
-    def get_initial_token(self) -> Optional[Any]:
+    def reset(self, reset_value: Optional[Any] = None) -> None:
         """
-        Get the page token that should be included in the request to get the first page of records
+        Reset the pagination's inner state
         """
 
     @abstractmethod
     def next_page_token(
-        self,
-        response: requests.Response,
-        last_page_size: int,
-        last_record: Optional[Record],
-        last_page_token_value: Optional[Any],
+        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
     ) -> Optional[Mapping[str, Any]]:
         """
         Returns the next_page_token to use to fetch the next page of records.
@@ -43,13 +39,12 @@ class Paginator(ABC, RequestOptionsProvider):
         :param response: the response to process
         :param last_page_size: the number of records read from the response
         :param last_record: the last record extracted from the response
-        :param last_page_token_value: The current value of the page token made on the last request
         :return: A mapping {"next_page_token": <token>} for the next page from the input response object. Returning None means there are no more pages to read in this response.
         """
         pass
 
     @abstractmethod
-    def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]:
+    def path(self) -> Optional[str]:
         """
         Returns the URL path to hit to fetch the next page of records
 

{airbyte_cdk-6.13.0.dev0 → airbyte_cdk-6.13.1.dev4101}/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py

@@ -43,6 +43,7 @@ class CursorPaginationStrategy(PaginationStrategy):
     )
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._initial_cursor = None
         if isinstance(self.cursor_value, str):
             self._cursor_value = InterpolatedString.create(self.cursor_value, parameters=parameters)
         else:
@@ -56,19 +57,10 @@ class CursorPaginationStrategy(PaginationStrategy):
 
     @property
     def initial_token(self) -> Optional[Any]:
-        """
-        CursorPaginationStrategy does not have an initial value because the next cursor is typically included
-        in the response of the first request. For Resumable Full Refresh streams that checkpoint the page
-        cursor, the next cursor should be read from the state or stream slice object.
-        """
-        return None
+        return self._initial_cursor
 
     def next_page_token(
-        self,
-        response: requests.Response,
-        last_page_size: int,
-        last_record: Optional[Record],
-        last_page_token_value: Optional[Any] = None,
+        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
     ) -> Optional[Any]:
         decoded_response = next(self.decoder.decode(response))
 
@@ -95,5 +87,8 @@ class CursorPaginationStrategy(PaginationStrategy):
         )
         return token if token else None
 
+    def reset(self, reset_value: Optional[Any] = None) -> None:
+        self._initial_cursor = reset_value
+
     def get_page_size(self) -> Optional[int]:
         return self.page_size

{airbyte_cdk-6.13.0.dev0 → airbyte_cdk-6.13.1.dev4101}/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py

@@ -52,6 +52,7 @@ class OffsetIncrement(PaginationStrategy):
     inject_on_first_request: bool = False
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._offset = 0
         page_size = str(self.page_size) if isinstance(self.page_size, int) else self.page_size
         if page_size:
             self._page_size: Optional[InterpolatedString] = InterpolatedString(
@@ -63,15 +64,11 @@ class OffsetIncrement(PaginationStrategy):
     @property
     def initial_token(self) -> Optional[Any]:
         if self.inject_on_first_request:
-            return 0
+            return self._offset
         return None
 
     def next_page_token(
-        self,
-        response: requests.Response,
-        last_page_size: int,
-        last_record: Optional[Record],
-        last_page_token_value: Optional[Any] = None,
+        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
     ) -> Optional[Any]:
         decoded_response = next(self.decoder.decode(response))
 
@@ -81,17 +78,9 @@ class OffsetIncrement(PaginationStrategy):
             and last_page_size < self._page_size.eval(self.config, response=decoded_response)
         ) or last_page_size == 0:
             return None
-        elif last_page_token_value is None:
-            # If the OffsetIncrement strategy does not inject on the first request, the incoming last_page_token_value
-            # will be None. For this case, we assume that None was the first page and progress to the next offset
-            return 0 + last_page_size
-        elif not isinstance(last_page_token_value, int):
-            raise ValueError(
-                "The page token for a OffsetIncrement pagination strategy must be an integer"
-            )
         else:
-            next_page_token_value = last_page_token_value + last_page_size
-            return next_page_token_value
+            self._offset += last_page_size
+            return self._offset
 
     def reset(self, reset_value: Optional[Any] = 0) -> None:
         if not isinstance(reset_value, int):

{airbyte_cdk-6.13.0.dev0 → airbyte_cdk-6.13.1.dev4101}/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py

@@ -31,6 +31,7 @@ class PageIncrement(PaginationStrategy):
     inject_on_first_request: bool = False
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._page = self.start_from_page
         if isinstance(self.page_size, int) or (self.page_size is None):
             self._page_size = self.page_size
         else:
@@ -42,30 +43,28 @@ class PageIncrement(PaginationStrategy):
     @property
     def initial_token(self) -> Optional[Any]:
         if self.inject_on_first_request:
-            return self.start_from_page
+            return self._page
         return None
 
     def next_page_token(
-        self,
-        response: requests.Response,
-        last_page_size: int,
-        last_record: Optional[Record],
-        last_page_token_value: Optional[Any],
+        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
     ) -> Optional[Any]:
         # Stop paginating when there are fewer records than the page size or the current page has no records
         if (self._page_size and last_page_size < self._page_size) or last_page_size == 0:
             return None
-        elif last_page_token_value is None:
-            # If the PageIncrement strategy does not inject on the first request, the incoming last_page_token_value
-            # may be None. When this is the case, we assume we've already requested the first page specified by
-            # start_from_page and must now get the next page
-            return self.start_from_page + 1
-        elif not isinstance(last_page_token_value, int):
+        else:
+            self._page += 1
+            return self._page
+
+    def reset(self, reset_value: Optional[Any] = None) -> None:
+        if reset_value is None:
+            self._page = self.start_from_page
+        elif not isinstance(reset_value, int):
             raise ValueError(
-                "The page token for a PageIncrement pagination strategy must be an integer"
+                f"Reset value {reset_value} for PageIncrement pagination strategy was not an integer"
             )
         else:
-            return last_page_token_value + 1
+            self._page = reset_value
 
     def get_page_size(self) -> Optional[int]:
         return self._page_size

{airbyte_cdk-6.13.0.dev0 → airbyte_cdk-6.13.1.dev4101}/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py

@@ -4,7 +4,7 @@
 
 from abc import abstractmethod
 from dataclasses import dataclass
-from typing import Any, Mapping, Optional
+from typing import Any, Optional
 
 import requests
 
@@ -26,21 +26,22 @@ class PaginationStrategy:
 
     @abstractmethod
     def next_page_token(
-        self,
-        response: requests.Response,
-        last_page_size: int,
-        last_record: Optional[Record],
-        last_page_token_value: Optional[Any],
+        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
     ) -> Optional[Any]:
         """
         :param response: response to process
         :param last_page_size: the number of records read from the response
         :param last_record: the last record extracted from the response
-        :param last_page_token_value: The current value of the page token made on the last request
         :return: next page token. Returns None if there are no more pages to fetch
         """
         pass
 
+    @abstractmethod
+    def reset(self, reset_value: Optional[Any] = None) -> None:
+        """
+        Reset the pagination's inner state
+        """
+
     @abstractmethod
     def get_page_size(self) -> Optional[int]:
         """

{airbyte_cdk-6.13.0.dev0 → airbyte_cdk-6.13.1.dev4101}/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py

@@ -44,19 +44,16 @@ class StopConditionPaginationStrategyDecorator(PaginationStrategy):
         self._stop_condition = stop_condition
 
     def next_page_token(
-        self,
-        response: requests.Response,
-        last_page_size: int,
-        last_record: Optional[Record],
-        last_page_token_value: Optional[Any] = None,
+        self, response: requests.Response, last_page_size: int, last_record: Optional[Record]
     ) -> Optional[Any]:
-        # We evaluate in reverse order because the assumption is that most of the APIs using data feed structure
-        # will return records in descending order. In terms of performance/memory, we return the records lazily
+        # We evaluate in reverse order because the assumption is that most of the APIs using data feed structure will return records in
+        # descending order. In terms of performance/memory, we return the records lazily
         if last_record and self._stop_condition.is_met(last_record):
             return None
-        return self._delegate.next_page_token(
-            response, last_page_size, last_record, last_page_token_value
-        )
+        return self._delegate.next_page_token(response, last_page_size, last_record)
+
+    def reset(self, reset_value: Optional[Any] = None) -> None:
+        self._delegate.reset(reset_value)
 
     def get_page_size(self) -> Optional[int]:
         return self._delegate.get_page_size()