airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py

@@ -3,13 +3,16 @@
 #
 
 from dataclasses import InitVar, dataclass
-from typing import Any, List, Mapping, Union
+from typing import Any, List, Mapping, Optional, Union
 
-import airbyte_cdk.sources.declarative.requesters.error_handlers.response_status as response_status
 import requests
-from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus
+
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    ErrorResolution,
+    ResponseAction,
+    create_fallback_error_resolution,
+)
 
 
 @dataclass
@@ -45,19 +48,32 @@ class CompositeErrorHandler(ErrorHandler):
             raise ValueError("CompositeErrorHandler expects at least 1 underlying error handler")
 
     @property
-    def max_retries(self) -> Union[int, None]:
+    def max_retries(self) -> Optional[int]:
         return self.error_handlers[0].max_retries
 
     @property
-    def max_time(self) -> Union[int, None]:
+    def max_time(self) -> Optional[int]:
         return max([error_handler.max_time or 0 for error_handler in self.error_handlers])
 
-    def interpret_response(self, response: requests.Response) -> ResponseStatus:
-        should_retry = ResponseStatus(ResponseAction.FAIL)
-        for retrier in self.error_handlers:
-            should_retry = retrier.interpret_response(response)
-            if should_retry.action == ResponseAction.SUCCESS:
-                return response_status.SUCCESS
-            if should_retry == response_status.IGNORE or should_retry.action == ResponseAction.RETRY:
-                return should_retry
-        return should_retry
+    def interpret_response(
+        self, response_or_exception: Optional[Union[requests.Response, Exception]]
+    ) -> ErrorResolution:
+        matched_error_resolution = None
+        for error_handler in self.error_handlers:
+            matched_error_resolution = error_handler.interpret_response(response_or_exception)
+
+            if not isinstance(matched_error_resolution, ErrorResolution):
+                continue
+
+            if matched_error_resolution.response_action == ResponseAction.SUCCESS:
+                return matched_error_resolution
+
+            if (
+                matched_error_resolution.response_action == ResponseAction.RETRY
+                or matched_error_resolution.response_action == ResponseAction.IGNORE
+            ):
+                return matched_error_resolution
+        if matched_error_resolution:
+            return matched_error_resolution
+
+        return create_fallback_error_resolution(response_or_exception)

airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py

@@ -5,17 +5,21 @@
 from dataclasses import InitVar, dataclass, field
 from typing import Any, List, Mapping, MutableMapping, Optional, Union
 
-import airbyte_cdk.sources.declarative.requesters.error_handlers.response_status as response_status
 import requests
-from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import (
-    ExponentialBackoffStrategy,
+
+from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import (
+    DefaultHttpResponseFilter,
+)
+from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import (
+    HttpResponseFilter,
 )
-from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy
-from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler
-from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import HttpResponseFilter
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy, ErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    SUCCESS_RESOLUTION,
+    ErrorResolution,
+    create_fallback_error_resolution,
+)
+from airbyte_cdk.sources.types import Config
 
 
 @dataclass
@@ -23,12 +27,12 @@ class DefaultErrorHandler(ErrorHandler):
     """
     Default error handler.
 
-    By default, the handler will only retry server errors (HTTP 5XX) and too many requests (HTTP 429) with exponential backoff.
+    By default, the handler will only use the `DEFAULT_ERROR_MAPPING` that is part of the Python CDK's `HttpStatusErrorHandler`.
 
-    If the response is successful, then return SUCCESS
+    If the response is successful, then a SUCCESS_RESOLUTION is returned.
     Otherwise, iterate over the response_filters.
     If any of the filter match the response, then return the appropriate status.
-    If the match is RETRY, then iterate sequentially over the backoff_strategies and return the first non-None backoff time.
+    When `DefaultErrorHandler.backoff_time()` is invoked, iterate sequentially over the backoff_strategies and return the first non-None backoff time, else return None.
 
     Sample configs:
 
@@ -89,8 +93,6 @@ class DefaultErrorHandler(ErrorHandler):
         to wait before retrying
     """
 
-    DEFAULT_BACKOFF_STRATEGY = ExponentialBackoffStrategy
-
     parameters: InitVar[Mapping[str, Any]]
     config: Config
     response_filters: Optional[List[HttpResponseFilter]] = None
@@ -101,57 +103,45 @@ class DefaultErrorHandler(ErrorHandler):
     backoff_strategies: Optional[List[BackoffStrategy]] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        self.response_filters = self.response_filters or []
-
         if not self.response_filters:
-            self.response_filters.append(
-                HttpResponseFilter(
-                    ResponseAction.RETRY, http_codes=HttpResponseFilter.DEFAULT_RETRIABLE_ERRORS, config=self.config, parameters={}
-                )
-            )
-            self.response_filters.append(HttpResponseFilter(ResponseAction.IGNORE, config={}, parameters={}))
-
-        if not self.backoff_strategies:
-            self.backoff_strategies = [DefaultErrorHandler.DEFAULT_BACKOFF_STRATEGY(parameters=parameters, config=self.config)]
+            self.response_filters = [HttpResponseFilter(config=self.config, parameters={})]
 
         self._last_request_to_attempt_count: MutableMapping[requests.PreparedRequest, int] = {}
 
-    @property  # type: ignore # overwrite the property to handle the case where max_retries is not provided in the constructor
-    def max_retries(self) -> Union[int, None]:
-        return self._max_retries
-
-    @max_retries.setter
-    def max_retries(self, value: int) -> None:
-        # Covers the case where max_retries is not provided in the constructor, which causes the property object
-        # to be set which we need to avoid doing
-        if not isinstance(value, property):
-            self._max_retries = value
-
-    def interpret_response(self, response: requests.Response) -> ResponseStatus:
-        request = response.request
-
-        if request not in self._last_request_to_attempt_count:
-            self._last_request_to_attempt_count = {request: 1}
-        else:
-            self._last_request_to_attempt_count[request] += 1
+    def interpret_response(
+        self, response_or_exception: Optional[Union[requests.Response, Exception]]
+    ) -> ErrorResolution:
         if self.response_filters:
             for response_filter in self.response_filters:
-                matched_status = response_filter.matches(
-                    response=response, backoff_time=self._backoff_time(response, self._last_request_to_attempt_count[request])
+                matched_error_resolution = response_filter.matches(
+                    response_or_exception=response_or_exception
                 )
-                if matched_status is not None:
-                    return matched_status
-
-        if response.ok:
-            return response_status.SUCCESS
-        # Fail if the response matches no filters
-        return response_status.FAIL
-
-    def _backoff_time(self, response: requests.Response, attempt_count: int) -> Optional[float]:
+                if matched_error_resolution:
+                    return matched_error_resolution
+        if isinstance(response_or_exception, requests.Response):
+            if response_or_exception.ok:
+                return SUCCESS_RESOLUTION
+
+        default_reponse_filter = DefaultHttpResponseFilter(parameters={}, config=self.config)
+        default_response_filter_resolution = default_reponse_filter.matches(response_or_exception)
+
+        return (
+            default_response_filter_resolution
+            if default_response_filter_resolution
+            else create_fallback_error_resolution(response_or_exception)
+        )
+
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int = 0,
+    ) -> Optional[float]:
         backoff = None
         if self.backoff_strategies:
-            for backoff_strategies in self.backoff_strategies:
-                backoff = backoff_strategies.backoff(response, attempt_count)
+            for backoff_strategy in self.backoff_strategies:
+                backoff = backoff_strategy.backoff_time(
+                    response_or_exception=response_or_exception, attempt_count=attempt_count
+                )
                 if backoff:
                     return backoff
         return backoff

airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import (
+    HttpResponseFilter,
+)
+from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import (
+    DEFAULT_ERROR_MAPPING,
+)
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    ErrorResolution,
+    create_fallback_error_resolution,
+)
+
+
+class DefaultHttpResponseFilter(HttpResponseFilter):
+    def matches(
+        self, response_or_exception: Optional[Union[requests.Response, Exception]]
+    ) -> Optional[ErrorResolution]:
+        default_mapped_error_resolution = None
+
+        if isinstance(response_or_exception, (requests.Response, Exception)):
+            mapped_key: Union[int, type] = (
+                response_or_exception.status_code
+                if isinstance(response_or_exception, requests.Response)
+                else response_or_exception.__class__
+            )
+
+            default_mapped_error_resolution = DEFAULT_ERROR_MAPPING.get(mapped_key)
+
+        return (
+            default_mapped_error_resolution
+            if default_mapped_error_resolution
+            else create_fallback_error_resolution(response_or_exception)
+        )

airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py

@@ -2,42 +2,16 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from abc import abstractmethod
+from abc import ABC
 from dataclasses import dataclass
-from typing import Union
 
-import requests
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
 
 
 @dataclass
-class ErrorHandler:
+class DeclarativeErrorHandler(ErrorHandler, ABC):
     """
-    Defines whether a request was successful and how to handle a failure.
-    """
-
-    @property
-    @abstractmethod
-    def max_retries(self) -> Union[int, None]:
-        """
-        Specifies maximum amount of retries for backoff policy. Return None for no limit.
-        """
-        pass
-
-    @property
-    @abstractmethod
-    def max_time(self) -> Union[int, None]:
-        """
-        Specifies maximum total waiting time (in seconds) for backoff policy. Return None for no limit.
-        """
-        pass
+    This interface exists to retain backwards compatability with connectors that reference the declarative ErrorHandler. As part of the effort to promote common interfaces to the Python CDK, this now extends the Python CDK ErrorHandler interface.
 
-    @abstractmethod
-    def interpret_response(self, response: requests.Response) -> ResponseStatus:
-        """
-        Evaluate response status describing whether a failing request should be retried or ignored.
-
-        :param response: response to evaluate
-        :return: response status
-        """
-        pass
+    `ErrorHandler` defines how to handle errors that occur during the request process, returning an ErrorResolution object that defines how to proceed.
+    """

airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py

@@ -6,96 +6,174 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Set, Union
 
 import requests
+
+from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction
-from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus
-from airbyte_cdk.sources.declarative.types import Config
-from airbyte_cdk.sources.streams.http.http import HttpStream
+from airbyte_cdk.sources.streams.http.error_handlers import JsonErrorMessageParser
+from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import (
+    DEFAULT_ERROR_MAPPING,
+)
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    ErrorResolution,
+    ResponseAction,
+)
+from airbyte_cdk.sources.types import Config
 
 
 @dataclass
 class HttpResponseFilter:
     """
-    Filter to select HttpResponses
+    Filter to select a response based on its HTTP status code, error message or a predicate.
+    If a response matches the filter, the response action, failure_type, and error message are returned as an ErrorResolution object.
+    For http_codes declared in the filter, the failure_type will default to `system_error`.
+    To override default failure_type use configured failure_type with ResponseAction.FAIL.
 
     Attributes:
         action (Union[ResponseAction, str]): action to execute if a request matches
+        failure_type (Union[ResponseAction, str]): failure type of traced exception if a response matches the filter
         http_codes (Set[int]): http code of matching requests
         error_message_contains (str): error substring of matching requests
         predicate (str): predicate to apply to determine if a request is matching
         error_message (Union[InterpolatedString, str): error message to display if the response matches the filter
     """
 
-    TOO_MANY_REQUESTS_ERRORS = {429}
-    DEFAULT_RETRIABLE_ERRORS = set([x for x in range(500, 600)]).union(TOO_MANY_REQUESTS_ERRORS)
-
-    action: Union[ResponseAction, str]
     config: Config
     parameters: InitVar[Mapping[str, Any]]
-    http_codes: Set[int] = None
-    error_message_contains: str = None
+    action: Optional[Union[ResponseAction, str]] = None
+    failure_type: Optional[Union[FailureType, str]] = None
+    http_codes: Optional[Set[int]] = None
+    error_message_contains: Optional[str] = None
     predicate: Union[InterpolatedBoolean, str] = ""
     error_message: Union[InterpolatedString, str] = ""
 
-    def __post_init__(self, parameters: Mapping[str, Any]):
-        if isinstance(self.action, str):
-            self.action = ResponseAction[self.action]
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if self.action is not None:
+            if (
+                self.http_codes is None
+                and self.predicate is None
+                and self.error_message_contains is None
+            ):
+                raise ValueError(
+                    "HttpResponseFilter requires a filter condition if an action is specified"
+                )
+            elif isinstance(self.action, str):
+                self.action = ResponseAction[self.action]
         self.http_codes = self.http_codes or set()
         if isinstance(self.predicate, str):
             self.predicate = InterpolatedBoolean(condition=self.predicate, parameters=parameters)
-        self.error_message = InterpolatedString.create(string_or_interpolated=self.error_message, parameters=parameters)
+        self.error_message = InterpolatedString.create(
+            string_or_interpolated=self.error_message, parameters=parameters
+        )
+        self._error_message_parser = JsonErrorMessageParser()
+        if self.failure_type and isinstance(self.failure_type, str):
+            self.failure_type = FailureType[self.failure_type]
+
+    def matches(
+        self, response_or_exception: Optional[Union[requests.Response, Exception]]
+    ) -> Optional[ErrorResolution]:
+        filter_action = self._matches_filter(response_or_exception)
+        mapped_key = (
+            response_or_exception.status_code
+            if isinstance(response_or_exception, requests.Response)
+            else response_or_exception.__class__
+        )
+
+        if isinstance(mapped_key, (int, Exception)):
+            default_mapped_error_resolution = self._match_default_error_mapping(mapped_key)
+        else:
+            default_mapped_error_resolution = None
 
-    def matches(self, response: requests.Response, backoff_time: Optional[float] = None) -> Optional[ResponseStatus]:
-        filter_action = self._matches_filter(response)
         if filter_action is not None:
-            error_message = self._create_error_message(response)
-            if filter_action == ResponseAction.RETRY:
-                return ResponseStatus(
-                    response_action=ResponseAction.RETRY,
-                    retry_in=backoff_time,
-                    error_message=error_message,
-                )
+            default_error_message = (
+                default_mapped_error_resolution.error_message
+                if default_mapped_error_resolution
+                else ""
+            )
+            error_message = None
+            if isinstance(response_or_exception, requests.Response):
+                error_message = self._create_error_message(response_or_exception)
+            error_message = error_message or default_error_message
+
+            if self.failure_type and filter_action == ResponseAction.FAIL:
+                failure_type = self.failure_type
+            elif default_mapped_error_resolution:
+                failure_type = default_mapped_error_resolution.failure_type
             else:
-                return ResponseStatus(filter_action, error_message=error_message)
+                failure_type = FailureType.system_error
+
+            return ErrorResolution(
+                response_action=filter_action,
+                failure_type=failure_type,
+                error_message=error_message,
+            )
+
+        if (
+            (isinstance(self.http_codes, list) and len(self.http_codes)) is None
+            and self.predicate is None
+            and self.error_message_contains is None
+        ) and default_mapped_error_resolution:
+            return default_mapped_error_resolution
+
         return None
 
-    def _matches_filter(self, response: requests.Response) -> Optional[ResponseAction]:
+    def _match_default_error_mapping(
+        self, mapped_key: Union[int, type[Exception]]
+    ) -> Optional[ErrorResolution]:
+        return DEFAULT_ERROR_MAPPING.get(mapped_key)
+
+    def _matches_filter(
+        self, response_or_exception: Optional[Union[requests.Response, Exception]]
+    ) -> Optional[ResponseAction]:
         """
-        Apply the filter on the response and return the action to execute if it matches
+        Apply the HTTP filter on the response and return the action to execute if it matches
         :param response: The HTTP response to evaluate
         :return: The action to execute. None if the response does not match the filter
         """
-        if (
-            response.status_code in self.http_codes
-            or (self._response_matches_predicate(response))
-            or (self._response_contains_error_message(response))
+        if isinstance(response_or_exception, requests.Response) and (
+            response_or_exception.status_code in self.http_codes  # type: ignore # http_codes set is always initialized to a value in __post_init__
+            or self._response_matches_predicate(response_or_exception)
+            or self._response_contains_error_message(response_or_exception)
         ):
-            return self.action
-        else:
-            return None
+            return self.action  # type: ignore # action is always cast to a ResponseAction not a str
+        return None
 
     @staticmethod
-    def _safe_response_json(response: requests.Response) -> dict:
+    def _safe_response_json(response: requests.Response) -> dict[str, Any]:
         try:
-            return response.json()
+            return response.json()  # type: ignore # Response.json() returns a dictionary even if the signature does not
         except requests.exceptions.JSONDecodeError:
             return {}
 
-    def _create_error_message(self, response: requests.Response) -> str:
+    def _create_error_message(self, response: requests.Response) -> Optional[str]:
         """
         Construct an error message based on the specified message template of the filter.
         :param response: The HTTP response which can be used during interpolation
         :return: The evaluated error message string to be emitted
         """
-        return self.error_message.eval(self.config, response=self._safe_response_json(response), headers=response.headers)
+        return self.error_message.eval(  # type: ignore [no-any-return, union-attr]
+            self.config, response=self._safe_response_json(response), headers=response.headers
+        )
 
     def _response_matches_predicate(self, response: requests.Response) -> bool:
-        return self.predicate and self.predicate.eval(None, response=self._safe_response_json(response), headers=response.headers)
+        return (
+            bool(
+                self.predicate.condition  # type: ignore [union-attr]
+                and self.predicate.eval(  # type: ignore [union-attr]
+                    None,  # type: ignore [arg-type]
+                    response=self._safe_response_json(response),
+                    headers=response.headers,
+                )
+            )
+            if self.predicate
+            else False
+        )
 
     def _response_contains_error_message(self, response: requests.Response) -> bool:
         if not self.error_message_contains:
             return False
         else:
-            error_message = HttpStream.parse_response_error_message(response)
-            return error_message and self.error_message_contains in error_message
+            error_message = self._error_message_parser.parse_response_error_message(
+                response=response
+            )
+            return bool(error_message and self.error_message_contains in error_message)