airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py

@@ -0,0 +1,327 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Mapping, MutableMapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.decoders import (
+    Decoder,
+    JsonDecoder,
+    PaginationDecoderDecorator,
+)
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
+from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
+    PaginationStrategy,
+)
+from airbyte_cdk.sources.declarative.requesters.request_option import (
+    RequestOption,
+    RequestOptionType,
+)
+from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
+from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
+from airbyte_cdk.utils.mapping_helpers import (
+    _validate_component_request_option_paths,
+    get_interpolation_context,
+)
+
+
+@dataclass
+class DefaultPaginator(Paginator):
+    """
+    Default paginator to request pages of results with a fixed size until the pagination strategy no longer returns a next_page_token
+
+    Examples:
+        1.
+        * fetches up to 10 records at a time by setting the "limit" request param to 10
+        * updates the request path with  "{{ response._metadata.next }}"
+        ```
+          paginator:
+            type: "DefaultPaginator"
+            page_size_option:
+              type: RequestOption
+              inject_into: request_parameter
+              field_name: limit
+            page_token_option:
+              type: RequestPath
+              path: "location"
+            pagination_strategy:
+              type: "CursorPagination"
+              cursor_value: "{{ response._metadata.next }}"
+              page_size: 10
+        ```
+
+        2.
+        * fetches up to 5 records at a time by setting the "page_size" header to 5
+        * increments a record counter and set the request parameter "offset" to the value of the counter
+        ```
+          paginator:
+            type: "DefaultPaginator"
+            page_size_option:
+              type: RequestOption
+              inject_into: header
+              field_name: page_size
+            pagination_strategy:
+              type: "OffsetIncrement"
+              page_size: 5
+            page_token_option:
+              option_type: "request_parameter"
+              field_name: "offset"
+        ```
+
+        3.
+        * fetches up to 5 records at a time by setting the "page_size" request param to 5
+        * increments a page counter and set the request parameter "page" to the value of the counter
+        ```
+          paginator:
+            type: "DefaultPaginator"
+            page_size_option:
+              type: RequestOption
+              inject_into: request_parameter
+              field_name: page_size
+            pagination_strategy:
+              type: "PageIncrement"
+              page_size: 5
+            page_token_option:
+              type: RequestOption
+              option_type: "request_parameter"
+              field_name: "page"
+        ```
+    Attributes:
+        page_size_option (Optional[RequestOption]): the request option to set the page size. Cannot be injected in the path.
+        page_token_option (Optional[RequestPath, RequestOption]): the request option to set the page token
+        pagination_strategy (PaginationStrategy): Strategy defining how to get the next page token
+        config (Config): connection config
+        url_base (Union[InterpolatedString, str]): endpoint's base url
+        decoder (Decoder): decoder to decode the response
+    """
+
+    pagination_strategy: PaginationStrategy
+    config: Config
+    url_base: Union[InterpolatedString, str]
+    parameters: InitVar[Mapping[str, Any]]
+    decoder: Decoder = field(
+        default_factory=lambda: PaginationDecoderDecorator(decoder=JsonDecoder(parameters={}))
+    )
+    page_size_option: Optional[RequestOption] = None
+    page_token_option: Optional[Union[RequestPath, RequestOption]] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if self.page_size_option and not self.pagination_strategy.get_page_size():
+            raise ValueError(
+                "page_size_option cannot be set if the pagination strategy does not have a page_size"
+            )
+        if isinstance(self.url_base, str):
+            self.url_base = InterpolatedString(string=self.url_base, parameters=parameters)
+
+        if self.page_token_option and not isinstance(self.page_token_option, RequestPath):
+            _validate_component_request_option_paths(
+                self.config,
+                self.page_size_option,
+                self.page_token_option,
+            )
+
+    def get_initial_token(self) -> Optional[Any]:
+        """
+        Return the page token that should be used for the first request of a stream
+
+        WARNING: get_initial_token() should not be used by streams that use RFR that perform checkpointing
+        of state using page numbers. Because paginators are stateless
+        """
+        return self.pagination_strategy.initial_token
+
+    def next_page_token(
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any] = None,
+    ) -> Optional[Mapping[str, Any]]:
+        next_page_token = self.pagination_strategy.next_page_token(
+            response=response,
+            last_page_size=last_page_size,
+            last_record=last_record,
+            last_page_token_value=last_page_token_value,
+        )
+        if next_page_token:
+            return {"next_page_token": next_page_token}
+        else:
+            return None
+
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
+        token = next_page_token.get("next_page_token") if next_page_token else None
+        if token and self.page_token_option and isinstance(self.page_token_option, RequestPath):
+            # make additional interpolation context
+            interpolation_context = get_interpolation_context(
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
+            )
+            # Replace url base to only return the path
+            return str(token).replace(self.url_base.eval(self.config, **interpolation_context), "")  # type: ignore # url_base is casted to a InterpolatedString in __post_init__
+        else:
+            return None
+
+    def get_request_params(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> MutableMapping[str, Any]:
+        return self._get_request_options(RequestOptionType.request_parameter, next_page_token)
+
+    def get_request_headers(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, str]:
+        return self._get_request_options(RequestOptionType.header, next_page_token)
+
+    def get_request_body_data(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return self._get_request_options(RequestOptionType.body_data, next_page_token)
+
+    def get_request_body_json(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return self._get_request_options(RequestOptionType.body_json, next_page_token)
+
+    def _get_request_options(
+        self, option_type: RequestOptionType, next_page_token: Optional[Mapping[str, Any]]
+    ) -> MutableMapping[str, Any]:
+        options: MutableMapping[str, Any] = {}
+
+        token = next_page_token.get("next_page_token") if next_page_token else None
+        if (
+            self.page_token_option
+            and token is not None
+            and isinstance(self.page_token_option, RequestOption)
+            and self.page_token_option.inject_into == option_type
+        ):
+            self.page_token_option.inject_into_request(options, token, self.config)
+
+        if (
+            self.page_size_option
+            and self.pagination_strategy.get_page_size()
+            and self.page_size_option.inject_into == option_type
+        ):
+            page_size = self.pagination_strategy.get_page_size()
+            self.page_size_option.inject_into_request(options, page_size, self.config)
+
+        return options
+
+
+class PaginatorTestReadDecorator(Paginator):
+    """
+    In some cases, we want to limit the number of requests that are made to the backend source. This class allows for limiting the number of
+    pages that are queried throughout a read command.
+
+    WARNING: This decorator is not currently thread-safe like the rest of the low-code framework because it has
+    an internal state to track the current number of pages counted so that it can exit early during a test read
+    """
+
+    _PAGE_COUNT_BEFORE_FIRST_NEXT_CALL = 1
+
+    def __init__(self, decorated: Paginator, maximum_number_of_pages: int = 5) -> None:
+        if maximum_number_of_pages and maximum_number_of_pages < 1:
+            raise ValueError(
+                f"The maximum number of pages on a test read needs to be strictly positive. Got {maximum_number_of_pages}"
+            )
+        self._maximum_number_of_pages = maximum_number_of_pages
+        self._decorated = decorated
+        self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL
+
+    def get_initial_token(self) -> Optional[Any]:
+        self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL
+        return self._decorated.get_initial_token()
+
+    def next_page_token(
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any] = None,
+    ) -> Optional[Mapping[str, Any]]:
+        if self._page_count >= self._maximum_number_of_pages:
+            return None
+
+        self._page_count += 1
+        return self._decorated.next_page_token(
+            response, last_page_size, last_record, last_page_token_value
+        )
+
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
+        return self._decorated.path(
+            next_page_token=next_page_token,
+            stream_state=stream_state,
+            stream_slice=stream_slice,
+        )
+
+    def get_request_params(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return self._decorated.get_request_params(
+            stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token
+        )
+
+    def get_request_headers(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, str]:
+        return self._decorated.get_request_headers(
+            stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token
+        )
+
+    def get_request_body_data(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Union[Mapping[str, Any], str]:
+        return self._decorated.get_request_body_data(
+            stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token
+        )
+
+    def get_request_body_json(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return self._decorated.get_request_body_json(
+            stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token
+        )

airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py

@@ -0,0 +1,76 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, MutableMapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
+from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
+
+
+@dataclass
+class NoPagination(Paginator):
+    """
+    Pagination implementation that never returns a next page.
+    """
+
+    parameters: InitVar[Mapping[str, Any]]
+
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
+        return None
+
+    def get_request_params(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> MutableMapping[str, Any]:
+        return {}
+
+    def get_request_headers(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, str]:
+        return {}
+
+    def get_request_body_data(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Union[Mapping[str, Any], str]:
+        return {}
+
+    def get_request_body_json(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_initial_token(self) -> Optional[Any]:
+        return None
+
+    def next_page_token(
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any],
+    ) -> Optional[Mapping[str, Any]]:
+        return {}

airbyte_cdk/sources/declarative/requesters/paginators/paginator.py

@@ -0,0 +1,65 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Mapping, Optional
+
+import requests
+
+from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
+    RequestOptionsProvider,
+)
+from airbyte_cdk.sources.types import Record, StreamSlice
+
+
+@dataclass
+class Paginator(ABC, RequestOptionsProvider):
+    """
+    Defines the token to use to fetch the next page of records from the API.
+
+    If needed, the Paginator will set request options to be set on the HTTP request to fetch the next page of records.
+    If the next_page_token is the path to the next page of records, then it should be accessed through the `path` method
+    """
+
+    @abstractmethod
+    def get_initial_token(self) -> Optional[Any]:
+        """
+        Get the page token that should be included in the request to get the first page of records
+        """
+
+    @abstractmethod
+    def next_page_token(
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any],
+    ) -> Optional[Mapping[str, Any]]:
+        """
+        Returns the next_page_token to use to fetch the next page of records.
+
+        :param response: the response to process
+        :param last_page_size: the number of records read from the response
+        :param last_record: the last record extracted from the response
+        :param last_page_token_value: The current value of the page token made on the last request
+        :return: A mapping {"next_page_token": <token>} for the next page from the input response object. Returning None means there are no more pages to read in this response.
+        """
+        pass
+
+    @abstractmethod
+    def path(
+        self,
+        next_page_token: Optional[Mapping[str, Any]],
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Optional[str]:
+        """
+        Returns the URL path to hit to fetch the next page of records
+
+        e.g: if you wanted to hit https://myapi.com/v1/some_entity then this will return "some_entity"
+
+        :return: path to hit to fetch the next request. Returning None means the path is not defined by the next_page_token
+        """
+        pass

airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.requesters.paginators.strategies.cursor_pagination_strategy import (
+    CursorPaginationStrategy,
+)
+from airbyte_cdk.sources.declarative.requesters.paginators.strategies.offset_increment import (
+    OffsetIncrement,
+)
+from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import (
+    PageIncrement,
+)
+from airbyte_cdk.sources.declarative.requesters.paginators.strategies.stop_condition import (
+    CursorStopCondition,
+    StopConditionPaginationStrategyDecorator,
+)
+
+__all__ = [
+    "CursorPaginationStrategy",
+    "CursorStopCondition",
+    "OffsetIncrement",
+    "PageIncrement",
+    "StopConditionPaginationStrategyDecorator",
+]

airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py

@@ -0,0 +1,98 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Dict, Mapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.decoders import (
+    Decoder,
+    JsonDecoder,
+    PaginationDecoderDecorator,
+)
+from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
+    PaginationStrategy,
+)
+from airbyte_cdk.sources.types import Config, Record
+
+
+@dataclass
+class CursorPaginationStrategy(PaginationStrategy):
+    """
+    Pagination strategy that evaluates an interpolated string to define the next page token
+
+    Attributes:
+        page_size (Optional[int]): the number of records to request
+        cursor_value (Union[InterpolatedString, str]): template string evaluating to the cursor value
+        config (Config): connection config
+        stop_condition (Optional[InterpolatedBoolean]): template string evaluating when to stop paginating
+        decoder (Decoder): decoder to decode the response
+    """
+
+    cursor_value: Union[InterpolatedString, str]
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+    page_size: Optional[int] = None
+    stop_condition: Optional[Union[InterpolatedBoolean, str]] = None
+    decoder: Decoder = field(
+        default_factory=lambda: PaginationDecoderDecorator(decoder=JsonDecoder(parameters={}))
+    )
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if isinstance(self.cursor_value, str):
+            self._cursor_value = InterpolatedString.create(self.cursor_value, parameters=parameters)
+        else:
+            self._cursor_value = self.cursor_value
+        if isinstance(self.stop_condition, str):
+            self._stop_condition: Optional[InterpolatedBoolean] = InterpolatedBoolean(
+                condition=self.stop_condition, parameters=parameters
+            )
+        else:
+            self._stop_condition = self.stop_condition
+
+    @property
+    def initial_token(self) -> Optional[Any]:
+        """
+        CursorPaginationStrategy does not have an initial value because the next cursor is typically included
+        in the response of the first request. For Resumable Full Refresh streams that checkpoint the page
+        cursor, the next cursor should be read from the state or stream slice object.
+        """
+        return None
+
+    def next_page_token(
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any] = None,
+    ) -> Optional[Any]:
+        decoded_response = next(self.decoder.decode(response))
+        # The default way that link is presented in requests.Response is a string of various links (last, next, etc). This
+        # is not indexable or useful for parsing the cursor, so we replace it with the link dictionary from response.links
+        headers: Dict[str, Any] = dict(response.headers)
+        headers["link"] = response.links
+        if self._stop_condition:
+            should_stop = self._stop_condition.eval(
+                self.config,
+                response=decoded_response,
+                headers=headers,
+                last_record=last_record,
+                last_page_size=last_page_size,
+            )
+            if should_stop:
+                return None
+        token = self._cursor_value.eval(
+            config=self.config,
+            response=decoded_response,
+            headers=headers,
+            last_record=last_record,
+            last_page_size=last_page_size,
+        )
+        return token if token else None
+
+    def get_page_size(self) -> Optional[int]:
+        return self.page_size

airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py

@@ -0,0 +1,102 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Mapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.decoders import (
+    Decoder,
+    JsonDecoder,
+    PaginationDecoderDecorator,
+)
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
+    PaginationStrategy,
+)
+from airbyte_cdk.sources.types import Config, Record
+
+
+@dataclass
+class OffsetIncrement(PaginationStrategy):
+    """
+    Pagination strategy that returns the number of records reads so far and returns it as the next page token
+    Examples:
+        # page_size to be a constant integer value
+        pagination_strategy:
+          type: OffsetIncrement
+          page_size: 2
+
+        # page_size to be a constant string value
+        pagination_strategy:
+          type: OffsetIncrement
+          page_size: "2"
+
+        # page_size to be an interpolated string value
+        pagination_strategy:
+          type: OffsetIncrement
+          page_size: "{{ parameters['items_per_page'] }}"
+
+    Attributes:
+        page_size (InterpolatedString): the number of records to request
+    """
+
+    config: Config
+    page_size: Optional[Union[str, int]]
+    parameters: InitVar[Mapping[str, Any]]
+    decoder: Decoder = field(
+        default_factory=lambda: PaginationDecoderDecorator(decoder=JsonDecoder(parameters={}))
+    )
+    inject_on_first_request: bool = False
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        page_size = str(self.page_size) if isinstance(self.page_size, int) else self.page_size
+        if page_size:
+            self._page_size: Optional[InterpolatedString] = InterpolatedString(
+                page_size, parameters=parameters
+            )
+        else:
+            self._page_size = None
+
+    @property
+    def initial_token(self) -> Optional[Any]:
+        if self.inject_on_first_request:
+            return 0
+        return None
+
+    def next_page_token(
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any] = None,
+    ) -> Optional[Any]:
+        decoded_response = next(self.decoder.decode(response))
+
+        # Stop paginating when there are fewer records than the page size or the current page has no records
+        if (
+            self._page_size
+            and last_page_size < self._page_size.eval(self.config, response=decoded_response)
+        ) or last_page_size == 0:
+            return None
+        elif last_page_token_value is None:
+            # If the OffsetIncrement strategy does not inject on the first request, the incoming last_page_token_value
+            # will be None. For this case, we assume that None was the first page and progress to the next offset
+            return 0 + last_page_size
+        elif not isinstance(last_page_token_value, int):
+            raise ValueError(
+                f"Last page token value {last_page_token_value} for OffsetIncrement pagination strategy was not an integer"
+            )
+        else:
+            return last_page_token_value + last_page_size
+
+    def get_page_size(self) -> Optional[int]:
+        if self._page_size:
+            page_size = self._page_size.eval(self.config)
+            if not isinstance(page_size, int):
+                raise Exception(f"{page_size} is of type {type(page_size)}. Expected {int}")
+            return page_size
+        else:
+            return None