airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py

@@ -0,0 +1,71 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
+    PaginationStrategy,
+)
+from airbyte_cdk.sources.types import Config, Record
+
+
+@dataclass
+class PageIncrement(PaginationStrategy):
+    """
+    Pagination strategy that returns the number of pages reads so far and returns it as the next page token
+
+    Attributes:
+        page_size (int): the number of records to request
+        start_from_page (int): number of the initial page
+    """
+
+    config: Config
+    page_size: Optional[Union[str, int]]
+    parameters: InitVar[Mapping[str, Any]]
+    start_from_page: int = 0
+    inject_on_first_request: bool = False
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if isinstance(self.page_size, int) or (self.page_size is None):
+            self._page_size = self.page_size
+        else:
+            page_size = InterpolatedString(self.page_size, parameters=parameters).eval(self.config)
+            if not isinstance(page_size, int):
+                raise Exception(f"{page_size} is of type {type(page_size)}. Expected {int}")
+            self._page_size = page_size
+
+    @property
+    def initial_token(self) -> Optional[Any]:
+        if self.inject_on_first_request:
+            return self.start_from_page
+        return None
+
+    def next_page_token(
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any],
+    ) -> Optional[Any]:
+        # Stop paginating when there are fewer records than the page size or the current page has no records
+        if (self._page_size and last_page_size < self._page_size) or last_page_size == 0:
+            return None
+        elif last_page_token_value is None:
+            # If the PageIncrement strategy does not inject on the first request, the incoming last_page_token_value
+            # may be None. When this is the case, we assume we've already requested the first page specified by
+            # start_from_page and must now get the next page
+            return self.start_from_page + 1
+        elif not isinstance(last_page_token_value, int):
+            raise ValueError(
+                f"Last page token value {last_page_token_value} for PageIncrement pagination strategy was not an integer"
+            )
+        else:
+            return last_page_token_value + 1
+
+    def get_page_size(self) -> Optional[int]:
+        return self._page_size

airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py

@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Any, Optional
+
+import requests
+
+from airbyte_cdk.sources.types import Record
+
+
+@dataclass
+class PaginationStrategy:
+    """
+    Defines how to get the next page token
+    """
+
+    @property
+    @abstractmethod
+    def initial_token(self) -> Optional[Any]:
+        """
+        Return the initial value of the token
+        """
+
+    @abstractmethod
+    def next_page_token(
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any],
+    ) -> Optional[Any]:
+        """
+        :param response: response to process
+        :param last_page_size: the number of records read from the response
+        :param last_record: the last record extracted from the response
+        :param last_page_token_value: The current value of the page token made on the last request
+        :return: next page token. Returns None if there are no more pages to fetch
+        """
+        pass
+
+    @abstractmethod
+    def get_page_size(self) -> Optional[int]:
+        """
+        :return: page size: The number of records to fetch in a page. Returns None if unspecified
+        """

airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py

@@ -0,0 +1,66 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+import requests
+
+from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
+from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
+    PaginationStrategy,
+)
+from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor
+from airbyte_cdk.sources.types import Record
+
+
+class PaginationStopCondition(ABC):
+    @abstractmethod
+    def is_met(self, record: Record) -> bool:
+        """
+        Given a condition is met, the pagination will stop
+
+        :param record: a record used to evaluate the condition
+        """
+        raise NotImplementedError()
+
+
+class CursorStopCondition(PaginationStopCondition):
+    def __init__(
+        self,
+        cursor: DeclarativeCursor
+        | ConcurrentCursor,  # migrate to use both old and concurrent versions
+    ):
+        self._cursor = cursor
+
+    def is_met(self, record: Record) -> bool:
+        return not self._cursor.should_be_synced(record)
+
+
+class StopConditionPaginationStrategyDecorator(PaginationStrategy):
+    def __init__(self, _delegate: PaginationStrategy, stop_condition: PaginationStopCondition):
+        self._delegate = _delegate
+        self._stop_condition = stop_condition
+
+    def next_page_token(
+        self,
+        response: requests.Response,
+        last_page_size: int,
+        last_record: Optional[Record],
+        last_page_token_value: Optional[Any] = None,
+    ) -> Optional[Any]:
+        # We evaluate in reverse order because the assumption is that most of the APIs using data feed structure
+        # will return records in descending order. In terms of performance/memory, we return the records lazily
+        if last_record and self._stop_condition.is_met(last_record):
+            return None
+        return self._delegate.next_page_token(
+            response, last_page_size, last_record, last_page_token_value
+        )
+
+    def get_page_size(self) -> Optional[int]:
+        return self._delegate.get_page_size()
+
+    @property
+    def initial_token(self) -> Optional[Any]:
+        return self._delegate.initial_token

airbyte_cdk/sources/declarative/requesters/request_option.py

@@ -0,0 +1,117 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from enum import Enum
+from typing import Any, List, Literal, Mapping, MutableMapping, Optional, Union
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.types import Config
+
+
+class RequestOptionType(Enum):
+    """
+    Describes where to set a value on a request
+    """
+
+    request_parameter = "request_parameter"
+    header = "header"
+    body_data = "body_data"
+    body_json = "body_json"
+
+
+@dataclass
+class RequestOption:
+    """
+    Describes an option to set on a request
+
+    Attributes:
+        field_name (str): Describes the name of the parameter to inject. Mutually exclusive with field_path.
+        field_path (list(str)): Describes the path to a nested field as a list of field names.
+          Only valid for body_json injection type, and mutually exclusive with field_name.
+        inject_into (RequestOptionType): Describes where in the HTTP request to inject the parameter
+    """
+
+    inject_into: RequestOptionType
+    parameters: InitVar[Mapping[str, Any]]
+    field_name: Optional[Union[InterpolatedString, str]] = None
+    field_path: Optional[List[Union[InterpolatedString, str]]] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        # Validate inputs. We should expect either field_name or field_path, but not both
+        if self.field_name is None and self.field_path is None:
+            raise ValueError("RequestOption requires either a field_name or field_path")
+
+        if self.field_name is not None and self.field_path is not None:
+            raise ValueError(
+                "Only one of field_name or field_path can be provided to RequestOption"
+            )
+
+        # Nested field injection is only supported for body JSON injection
+        if self.field_path is not None and self.inject_into != RequestOptionType.body_json:
+            raise ValueError(
+                "Nested field injection is only supported for body JSON injection. Please use a top-level field_name for other injection types."
+            )
+
+        # Convert field_name and field_path into InterpolatedString objects if they are strings
+        if self.field_name is not None:
+            self.field_name = InterpolatedString.create(self.field_name, parameters=parameters)
+        elif self.field_path is not None:
+            self.field_path = [
+                InterpolatedString.create(segment, parameters=parameters)
+                for segment in self.field_path
+            ]
+
+    @property
+    def _is_field_path(self) -> bool:
+        """Returns whether this option is a field path (ie, a nested field)"""
+        return self.field_path is not None
+
+    def inject_into_request(
+        self,
+        target: MutableMapping[str, Any],
+        value: Any,
+        config: Config,
+    ) -> None:
+        """
+        Inject a request option value into a target request structure using either field_name or field_path.
+        For non-body-json injection, only top-level field names are supported.
+        For body-json injection, both field names and nested field paths are supported.
+
+        Args:
+            target: The request structure to inject the value into
+            value: The value to inject
+            config: The config object to use for interpolation
+        """
+        if self._is_field_path:
+            if self.inject_into != RequestOptionType.body_json:
+                raise ValueError(
+                    "Nested field injection is only supported for body JSON injection. Please use a top-level field_name for other injection types."
+                )
+
+            assert self.field_path is not None  # for type checker
+            current = target
+            # Convert path segments into strings, evaluating any interpolated segments
+            # Example: ["data", "{{ config[user_type] }}", "id"] -> ["data", "admin", "id"]
+            *path_parts, final_key = [
+                str(
+                    segment.eval(config=config)
+                    if isinstance(segment, InterpolatedString)
+                    else segment
+                )
+                for segment in self.field_path
+            ]
+
+            # Build a nested dictionary structure and set the final value at the deepest level
+            for part in path_parts:
+                current = current.setdefault(part, {})
+            current[final_key] = value
+        else:
+            # For non-nested fields, evaluate the field name if it's an interpolated string
+            key = (
+                self.field_name.eval(config=config)
+                if isinstance(self.field_name, InterpolatedString)
+                else self.field_name
+            )
+            target[str(key)] = value

airbyte_cdk/sources/declarative/requesters/request_options/__init__.py

@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.requesters.request_options.datetime_based_request_options_provider import (
+    DatetimeBasedRequestOptionsProvider,
+)
+from airbyte_cdk.sources.declarative.requesters.request_options.default_request_options_provider import (
+    DefaultRequestOptionsProvider,
+)
+from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider import (
+    InterpolatedRequestOptionsProvider,
+)
+from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
+    RequestOptionsProvider,
+)
+
+__all__ = [
+    "DatetimeBasedRequestOptionsProvider",
+    "DefaultRequestOptionsProvider",
+    "InterpolatedRequestOptionsProvider",
+    "RequestOptionsProvider",
+]

airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py

@@ -0,0 +1,92 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, MutableMapping, Optional, Union
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.requesters.request_option import (
+    RequestOption,
+    RequestOptionType,
+)
+from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
+    RequestOptionsProvider,
+)
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class DatetimeBasedRequestOptionsProvider(RequestOptionsProvider):
+    """
+    Request options provider that extracts fields from the stream_slice and injects them into the respective location in the
+    outbound request being made
+    """
+
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+    start_time_option: Optional[RequestOption] = None
+    end_time_option: Optional[RequestOption] = None
+    partition_field_start: Optional[str] = None
+    partition_field_end: Optional[str] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._partition_field_start = InterpolatedString.create(
+            self.partition_field_start or "start_time", parameters=parameters
+        )
+        self._partition_field_end = InterpolatedString.create(
+            self.partition_field_end or "end_time", parameters=parameters
+        )
+
+    def get_request_params(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return self._get_request_options(RequestOptionType.request_parameter, stream_slice)
+
+    def get_request_headers(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return self._get_request_options(RequestOptionType.header, stream_slice)
+
+    def get_request_body_data(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Union[Mapping[str, Any], str]:
+        return self._get_request_options(RequestOptionType.body_data, stream_slice)
+
+    def get_request_body_json(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return self._get_request_options(RequestOptionType.body_json, stream_slice)
+
+    def _get_request_options(
+        self, option_type: RequestOptionType, stream_slice: Optional[StreamSlice]
+    ) -> Mapping[str, Any]:
+        options: MutableMapping[str, Any] = {}
+        if not stream_slice:
+            return options
+
+        if self.start_time_option and self.start_time_option.inject_into == option_type:
+            start_time_value = stream_slice.get(self._partition_field_start.eval(self.config))
+            self.start_time_option.inject_into_request(options, start_time_value, self.config)
+
+        if self.end_time_option and self.end_time_option.inject_into == option_type:
+            end_time_value = stream_slice.get(self._partition_field_end.eval(self.config))
+            self.end_time_option.inject_into_request(options, end_time_value, self.config)
+
+        return options

airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py

@@ -0,0 +1,60 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, Optional, Union
+
+from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
+    RequestOptionsProvider,
+)
+from airbyte_cdk.sources.types import StreamSlice, StreamState
+
+
+@dataclass
+class DefaultRequestOptionsProvider(RequestOptionsProvider):
+    """
+    Request options provider that extracts fields from the stream_slice and injects them into the respective location in the
+    outbound request being made
+    """
+
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        pass
+
+    def get_request_params(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_request_headers(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_request_body_data(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Union[Mapping[str, Any], str]:
+        return {}
+
+    def get_request_body_json(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}

airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py

@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Mapping, Optional, Union
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping import (
+    InterpolatedNestedMapping,
+    NestedMapping,
+)
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.types import Config, StreamSlice
+
+
+@dataclass
+class InterpolatedNestedRequestInputProvider:
+    """
+    Helper class that generically performs string interpolation on a provided deeply nested dictionary or string input
+    """
+
+    parameters: InitVar[Mapping[str, Any]]
+    request_inputs: Optional[Union[str, NestedMapping]] = field(default=None)
+    config: Config = field(default_factory=dict)
+    _interpolator: Optional[Union[InterpolatedString, InterpolatedNestedMapping]] = field(
+        init=False, repr=False, default=None
+    )
+    _request_inputs: Optional[Union[str, NestedMapping]] = field(
+        init=False, repr=False, default=None
+    )
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._request_inputs = self.request_inputs or {}
+        if isinstance(self._request_inputs, str):
+            self._interpolator = InterpolatedString(
+                self._request_inputs, default="", parameters=parameters
+            )
+        else:
+            self._interpolator = InterpolatedNestedMapping(
+                self._request_inputs, parameters=parameters
+            )
+
+    def eval_request_inputs(
+        self,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        """
+        Returns the request inputs to set on an outgoing HTTP request
+
+        :param stream_slice: The stream slice
+        :param next_page_token: The pagination token
+        :return: The request inputs to set on an outgoing HTTP request
+        """
+        kwargs = {
+            "stream_slice": stream_slice,
+            "next_page_token": next_page_token,
+        }
+        return self._interpolator.eval(self.config, **kwargs)  # type: ignore  # self._interpolator is always initialized with a value and will not be None

airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py

@@ -0,0 +1,68 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Mapping, Optional, Tuple, Type, Union
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+
+@dataclass
+class InterpolatedRequestInputProvider:
+    """
+    Helper class that generically performs string interpolation on the provided dictionary or string input
+    """
+
+    parameters: InitVar[Mapping[str, Any]]
+    request_inputs: Optional[Union[str, Mapping[str, str]]] = field(default=None)
+    config: Config = field(default_factory=dict)
+    _interpolator: Optional[Union[InterpolatedString, InterpolatedMapping]] = field(
+        init=False, repr=False, default=None
+    )
+    _request_inputs: Optional[Union[str, Mapping[str, str]]] = field(
+        init=False, repr=False, default=None
+    )
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._request_inputs = self.request_inputs or {}
+        if isinstance(self._request_inputs, str):
+            self._interpolator = InterpolatedString(
+                self._request_inputs, default="", parameters=parameters
+            )
+        else:
+            self._interpolator = InterpolatedMapping(self._request_inputs, parameters=parameters)
+
+    def eval_request_inputs(
+        self,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+        valid_key_types: Optional[Tuple[Type[Any]]] = None,
+        valid_value_types: Optional[Tuple[Type[Any], ...]] = None,
+    ) -> Mapping[str, Any]:
+        """
+        Returns the request inputs to set on an outgoing HTTP request
+
+        :param stream_slice: The stream slice
+        :param next_page_token: The pagination token
+        :param valid_key_types: A tuple of types that the interpolator should allow
+        :param valid_value_types: A tuple of types that the interpolator should allow
+        :return: The request inputs to set on an outgoing HTTP request
+        """
+        kwargs = {
+            "stream_slice": stream_slice,
+            "next_page_token": next_page_token,
+        }
+        interpolated_value = self._interpolator.eval(  # type: ignore # self._interpolator is always initialized with a value and will not be None
+            self.config,
+            valid_key_types=valid_key_types,
+            valid_value_types=valid_value_types,
+            **kwargs,
+        )
+
+        if isinstance(interpolated_value, dict):
+            non_null_tokens = {k: v for k, v in interpolated_value.items() if v is not None}
+            return non_null_tokens
+        return interpolated_value  # type: ignore[no-any-return]