airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py

@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import numbers
+from re import Pattern
+from typing import Optional
+
+import requests
+
+
+def get_numeric_value_from_header(
+    response: requests.Response, header: str, regex: Optional[Pattern[str]]
+) -> Optional[float]:
+    """
+    Extract a header value from the response as a float
+    :param response: response the extract header value from
+    :param header: Header to extract
+    :param regex: optional regex to apply on the header to obtain the value
+    :return: header value as float if it's a number. None otherwise
+    """
+    header_value = response.headers.get(header, None)
+    if not header_value:
+        return None
+    if isinstance(header_value, str):
+        if regex:
+            match = regex.match(header_value)
+            if match:
+                header_value = match.group()
+        return _as_float(header_value)
+    elif isinstance(header_value, numbers.Number):
+        return float(header_value)  # type: ignore[arg-type]
+    else:
+        return None
+
+
+def _as_float(s: str) -> Optional[float]:
+    try:
+        return float(s)
+    except ValueError:
+        return None

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py

@@ -0,0 +1,70 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import re
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
+    get_numeric_value_from_header,
+)
+from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import (
+    BackoffStrategy,
+)
+from airbyte_cdk.sources.types import Config
+from airbyte_cdk.utils import AirbyteTracedException
+
+
+@dataclass
+class WaitTimeFromHeaderBackoffStrategy(BackoffStrategy):
+    """
+    Extract wait time from http header
+
+    Attributes:
+        header (str): header to read wait time from
+        regex (Optional[str]): optional regex to apply on the header to extract its value
+        max_waiting_time_in_seconds: (Optional[float]): given the value extracted from the header is greater than this value, stop the stream
+    """
+
+    header: Union[InterpolatedString, str]
+    parameters: InitVar[Mapping[str, Any]]
+    config: Config
+    regex: Optional[Union[InterpolatedString, str]] = None
+    max_waiting_time_in_seconds: Optional[float] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self.regex = (
+            InterpolatedString.create(self.regex, parameters=parameters) if self.regex else None
+        )
+        self.header = InterpolatedString.create(self.header, parameters=parameters)
+
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int,
+    ) -> Optional[float]:
+        header = self.header.eval(config=self.config)  # type: ignore  # header is always cast to an interpolated stream
+        if self.regex:
+            evaled_regex = self.regex.eval(self.config)  # type: ignore # header is always cast to an interpolated string
+            regex = re.compile(evaled_regex)
+        else:
+            regex = None
+        header_value = None
+        if isinstance(response_or_exception, requests.Response):
+            header_value = get_numeric_value_from_header(response_or_exception, header, regex)
+            if (
+                self.max_waiting_time_in_seconds
+                and header_value
+                and header_value >= self.max_waiting_time_in_seconds
+            ):
+                raise AirbyteTracedException(
+                    internal_message=f"Rate limit wait time {header_value} is greater than max waiting time of {self.max_waiting_time_in_seconds} seconds. Stopping the stream...",
+                    message="The rate limit is greater than max waiting time has been reached.",
+                    failure_type=FailureType.transient_error,
+                )
+        return header_value

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py

@@ -0,0 +1,77 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import numbers
+import re
+import time
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
+    get_numeric_value_from_header,
+)
+from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import (
+    BackoffStrategy,
+)
+from airbyte_cdk.sources.types import Config
+
+
+@dataclass
+class WaitUntilTimeFromHeaderBackoffStrategy(BackoffStrategy):
+    """
+    Extract time at which we can retry the request from response header
+    and wait for the difference between now and that time
+
+    Attributes:
+        header (str): header to read wait time from
+        min_wait (Optional[float]): minimum time to wait for safety
+        regex (Optional[str]): optional regex to apply on the header to extract its value
+    """
+
+    header: Union[InterpolatedString, str]
+    parameters: InitVar[Mapping[str, Any]]
+    config: Config
+    min_wait: Optional[Union[float, InterpolatedString, str]] = None
+    regex: Optional[Union[InterpolatedString, str]] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self.header = InterpolatedString.create(self.header, parameters=parameters)
+        self.regex = (
+            InterpolatedString.create(self.regex, parameters=parameters) if self.regex else None
+        )
+        if not isinstance(self.min_wait, InterpolatedString):
+            self.min_wait = InterpolatedString.create(str(self.min_wait), parameters=parameters)
+
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int,
+    ) -> Optional[float]:
+        now = time.time()
+        header = self.header.eval(self.config)  # type: ignore # header is always cast to an interpolated string
+        if self.regex:
+            evaled_regex = self.regex.eval(self.config)  # type: ignore # header is always cast to an interpolated string
+            regex = re.compile(evaled_regex)
+        else:
+            regex = None
+        wait_until = None
+        if isinstance(response_or_exception, requests.Response):
+            wait_until = get_numeric_value_from_header(response_or_exception, header, regex)
+        min_wait = self.min_wait.eval(self.config)  # type: ignore # header is always cast to an interpolated string
+        if wait_until is None or not wait_until:
+            return float(min_wait) if min_wait else None
+        if (isinstance(wait_until, str) and wait_until.isnumeric()) or isinstance(
+            wait_until, numbers.Number
+        ):
+            wait_time = float(wait_until) - now
+        else:
+            return float(min_wait)
+        if min_wait:
+            return float(max(wait_time, min_wait))
+        elif wait_time < 0:
+            return None
+        return wait_time

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py

@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC
+from dataclasses import dataclass
+
+from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
+
+
+@dataclass
+class DecalarativeBackoffStrategy(BackoffStrategy, ABC):
+    """
+    This interface exists to retain backwards compatability with connectors that reference the declarative BackoffStrategy. As part of the effort to promote common interfaces to the Python CDK, this now extends the Python CDK backoff strategy interface.
+
+    Backoff strategy defining how long to wait before retrying a request that resulted in an error.
+    """

airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py

@@ -0,0 +1,101 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, List, Mapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers.backoff_strategy import BackoffStrategy
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    ErrorResolution,
+    ResponseAction,
+    create_fallback_error_resolution,
+)
+
+
+@dataclass
+class CompositeErrorHandler(ErrorHandler):
+    """
+    Error handler that sequentially iterates over a list of `ErrorHandler`s
+
+    Sample config chaining 2 different retriers:
+        error_handler:
+          type: "CompositeErrorHandler"
+          error_handlers:
+            - response_filters:
+                - predicate: "{{ 'codase' in response }}"
+                  action: RETRY
+              backoff_strategies:
+                - type: "ConstantBackoff"
+                  backoff_time_in_seconds: 5
+            - response_filters:
+                - http_codes: [ 403 ]
+                  action: RETRY
+              backoff_strategies:
+                - type: "ConstantBackoff"
+                  backoff_time_in_seconds: 10
+    Attributes:
+        error_handlers (List[ErrorHandler]): list of error handlers
+    """
+
+    error_handlers: List[ErrorHandler]
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if not self.error_handlers:
+            raise ValueError("CompositeErrorHandler expects at least 1 underlying error handler")
+
+    @property
+    def max_retries(self) -> Optional[int]:
+        return self.error_handlers[0].max_retries
+
+    @property
+    def max_time(self) -> Optional[int]:
+        return max([error_handler.max_time or 0 for error_handler in self.error_handlers])
+
+    def interpret_response(
+        self, response_or_exception: Optional[Union[requests.Response, Exception]]
+    ) -> ErrorResolution:
+        matched_error_resolution = None
+        for error_handler in self.error_handlers:
+            matched_error_resolution = error_handler.interpret_response(response_or_exception)
+
+            if not isinstance(matched_error_resolution, ErrorResolution):
+                continue
+
+            if matched_error_resolution.response_action == ResponseAction.SUCCESS:
+                return matched_error_resolution
+
+            if (
+                matched_error_resolution.response_action == ResponseAction.RETRY
+                or matched_error_resolution.response_action == ResponseAction.IGNORE
+            ):
+                return matched_error_resolution
+        if matched_error_resolution:
+            return matched_error_resolution
+
+        return create_fallback_error_resolution(response_or_exception)
+
+    @property
+    def backoff_strategies(self) -> Optional[List[BackoffStrategy]]:
+        """
+        Combines backoff strategies from all child error handlers into a single flattened list.
+
+        When used with HttpRequester, note the following behavior:
+        - In HttpRequester.__post_init__, the entire list of backoff strategies is assigned to the error handler
+        - However, the error handler's backoff_time() method only ever uses the first non-None strategy in the list
+        - This means that if any backoff strategies are present, the first non-None strategy becomes the default
+        - This applies to both user-defined response filters and errors from DEFAULT_ERROR_MAPPING
+        - The list structure is not used to map different strategies to different error conditions
+        - Therefore, subsequent strategies in the list will not be used
+
+        Returns None if no handlers have strategies defined, which will result in HttpRequester using its default backoff strategy.
+        """
+        all_strategies = []
+        for handler in self.error_handlers:
+            if hasattr(handler, "backoff_strategies") and handler.backoff_strategies:
+                all_strategies.extend(handler.backoff_strategies)
+        return all_strategies if all_strategies else None

airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py

@@ -0,0 +1,147 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from typing import Any, List, Mapping, MutableMapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import (
+    DefaultHttpResponseFilter,
+)
+from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import (
+    HttpResponseFilter,
+)
+from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy, ErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    SUCCESS_RESOLUTION,
+    ErrorResolution,
+    create_fallback_error_resolution,
+)
+from airbyte_cdk.sources.types import Config
+
+
+@dataclass
+class DefaultErrorHandler(ErrorHandler):
+    """
+    Default error handler.
+
+    By default, the handler will only use the `DEFAULT_ERROR_MAPPING` that is part of the Python CDK's `HttpStatusErrorHandler`.
+
+    If the response is successful, then a SUCCESS_RESOLUTION is returned.
+    Otherwise, iterate over the response_filters.
+    If any of the filter match the response, then return the appropriate status.
+    When `DefaultErrorHandler.backoff_time()` is invoked, iterate sequentially over the backoff_strategies and return the first non-None backoff time, else return None.
+
+    Sample configs:
+
+    1. retry 10 times
+    `
+        error_handler:
+          max_retries: 10
+    `
+    2. backoff for 5 seconds
+    `
+        error_handler:
+          backoff_strategies:
+            - type: "ConstantBackoff"
+              backoff_time_in_seconds: 5
+    `
+    3. retry on HTTP 404
+    `
+        error_handler:
+          response_filters:
+            - http_codes: [ 404 ]
+              action: RETRY
+    `
+    4. ignore HTTP 404
+    `
+      error_handler:
+        response_filters:
+          - http_codes: [ 404 ]
+            action: IGNORE
+    `
+    5. retry if error message contains `retrythisrequest!` substring
+    `
+        error_handler:
+          response_filters:
+            - error_message_contain: "retrythisrequest!"
+              action: IGNORE
+    `
+    6. retry if 'code' is a field present in the response body
+    `
+        error_handler:
+          response_filters:
+            - predicate: "{{ 'code' in response }}"
+              action: IGNORE
+    `
+
+    7. ignore 429 and retry on 404
+    `
+        error_handler:
+        - http_codes: [ 429 ]
+          action: IGNORE
+        - http_codes: [ 404 ]
+          action: RETRY
+    `
+
+    Attributes:
+        response_filters (Optional[List[HttpResponseFilter]]): response filters to iterate on
+        max_retries (Optional[int]): maximum retry attempts
+        backoff_strategies (Optional[List[BackoffStrategy]]): list of backoff strategies to use to determine how long
+        to wait before retrying
+    """
+
+    parameters: InitVar[Mapping[str, Any]]
+    config: Config
+    response_filters: Optional[List[HttpResponseFilter]] = None
+    max_retries: Optional[int] = 5
+    max_time: int = 60 * 10
+    _max_retries: int = field(init=False, repr=False, default=5)
+    _max_time: int = field(init=False, repr=False, default=60 * 10)
+    backoff_strategies: Optional[List[BackoffStrategy]] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if not self.response_filters:
+            self.response_filters = [HttpResponseFilter(config=self.config, parameters={})]
+
+        self._last_request_to_attempt_count: MutableMapping[requests.PreparedRequest, int] = {}
+
+    def interpret_response(
+        self, response_or_exception: Optional[Union[requests.Response, Exception]]
+    ) -> ErrorResolution:
+        if self.response_filters:
+            for response_filter in self.response_filters:
+                matched_error_resolution = response_filter.matches(
+                    response_or_exception=response_or_exception
+                )
+                if matched_error_resolution:
+                    return matched_error_resolution
+        if isinstance(response_or_exception, requests.Response):
+            if response_or_exception.ok:
+                return SUCCESS_RESOLUTION
+
+        default_reponse_filter = DefaultHttpResponseFilter(parameters={}, config=self.config)
+        default_response_filter_resolution = default_reponse_filter.matches(response_or_exception)
+
+        return (
+            default_response_filter_resolution
+            if default_response_filter_resolution
+            else create_fallback_error_resolution(response_or_exception)
+        )
+
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int = 0,
+    ) -> Optional[float]:
+        backoff = None
+        if self.backoff_strategies:
+            for backoff_strategy in self.backoff_strategies:
+                backoff = backoff_strategy.backoff_time(
+                    response_or_exception=response_or_exception, attempt_count=attempt_count
+                )
+                if backoff:
+                    return backoff
+        return backoff

airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import (
+    HttpResponseFilter,
+)
+from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import (
+    DEFAULT_ERROR_MAPPING,
+)
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    ErrorResolution,
+    create_fallback_error_resolution,
+)
+
+
+class DefaultHttpResponseFilter(HttpResponseFilter):
+    def matches(
+        self, response_or_exception: Optional[Union[requests.Response, Exception]]
+    ) -> Optional[ErrorResolution]:
+        default_mapped_error_resolution = None
+
+        if isinstance(response_or_exception, (requests.Response, Exception)):
+            mapped_key: Union[int, type] = (
+                response_or_exception.status_code
+                if isinstance(response_or_exception, requests.Response)
+                else response_or_exception.__class__
+            )
+
+            default_mapped_error_resolution = DEFAULT_ERROR_MAPPING.get(mapped_key)
+
+        return (
+            default_mapped_error_resolution
+            if default_mapped_error_resolution
+            else create_fallback_error_resolution(response_or_exception)
+        )

airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py

@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC
+from dataclasses import dataclass
+
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
+
+
+@dataclass
+class DeclarativeErrorHandler(ErrorHandler, ABC):
+    """
+    This interface exists to retain backwards compatability with connectors that reference the declarative ErrorHandler. As part of the effort to promote common interfaces to the Python CDK, this now extends the Python CDK ErrorHandler interface.
+
+    `ErrorHandler` defines how to handle errors that occur during the request process, returning an ErrorResolution object that defines how to proceed.
+    """