airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

airbyte_cdk/sources/streams/http/availability_strategy.py

@@ -4,20 +4,20 @@
 
 import logging
 import typing
-from typing import Dict, Optional, Tuple
+from typing import Optional, Tuple
 
-import requests
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
-from airbyte_cdk.sources.streams.utils.stream_helper import get_first_record_for_slice, get_first_stream_slice
-from requests import HTTPError
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 if typing.TYPE_CHECKING:
     from airbyte_cdk.sources import Source
 
 
 class HttpAvailabilityStrategy(AvailabilityStrategy):
-    def check_availability(self, stream: Stream, logger: logging.Logger, source: Optional["Source"]) -> Tuple[bool, Optional[str]]:
+    def check_availability(
+        self, stream: Stream, logger: logging.Logger, source: Optional["Source"] = None
+    ) -> Tuple[bool, Optional[str]]:
         """
         Check stream availability by attempting to read the first record of the
         stream.
@@ -30,114 +30,25 @@ class HttpAvailabilityStrategy(AvailabilityStrategy):
           for some reason and the str should describe what went wrong and how to
           resolve the unavailability, if possible.
         """
+        reason: Optional[str]
         try:
             # Some streams need a stream slice to read records (e.g. if they have a SubstreamPartitionRouter)
             # Streams that don't need a stream slice will return `None` as their first stream slice.
-            stream_slice = get_first_stream_slice(stream)
+            stream_slice = self.get_first_stream_slice(stream)
         except StopIteration:
             # If stream_slices has no `next()` item (Note - this is different from stream_slices returning [None]!)
             # This can happen when a substream's `stream_slices` method does a `for record in parent_records: yield <something>`
             # without accounting for the case in which the parent stream is empty.
             reason = f"Cannot attempt to connect to stream {stream.name} - no stream slices were found, likely because the parent stream is empty."
             return False, reason
-        except HTTPError as error:
-            is_available, reason = self.handle_http_error(stream, logger, source, error)
-            if not is_available:
-                reason = f"Unable to get slices for {stream.name} stream, because of error in parent stream. {reason}"
-            return is_available, reason
+        except AirbyteTracedException as error:
+            return False, error.message
 
         try:
-            get_first_record_for_slice(stream, stream_slice)
+            self.get_first_record_for_slice(stream, stream_slice)
             return True, None
         except StopIteration:
             logger.info(f"Successfully connected to stream {stream.name}, but got 0 records.")
             return True, None
-        except HTTPError as error:
-            is_available, reason = self.handle_http_error(stream, logger, source, error)
-            if not is_available:
-                reason = f"Unable to read {stream.name} stream. {reason}"
-            return is_available, reason
-
-    def handle_http_error(
-        self, stream: Stream, logger: logging.Logger, source: Optional["Source"], error: HTTPError
-    ) -> Tuple[bool, Optional[str]]:
-        """
-        Override this method to define error handling for various `HTTPError`s
-        that are raised while attempting to check a stream's availability.
-
-        Checks whether an error's status_code is in a list of unavailable_error_codes,
-        and gets the associated reason for that error.
-
-        :param stream: stream
-        :param logger: source logger
-        :param source: optional (source)
-        :param error: HTTPError raised while checking stream's availability.
-        :return: A tuple of (boolean, str). If boolean is true, then the stream
-          is available, and no str is required. Otherwise, the stream is unavailable
-          for some reason and the str should describe what went wrong and how to
-          resolve the unavailability, if possible.
-        """
-        status_code = error.response.status_code
-        known_status_codes = self.reasons_for_unavailable_status_codes(stream, logger, source, error)
-        known_reason = known_status_codes.get(status_code)
-        if not known_reason:
-            # If the HTTPError is not in the dictionary of errors we know how to handle, don't except
-            raise error
-
-        doc_ref = self._visit_docs_message(logger, source)
-        reason = f"The endpoint {error.response.url} returned {status_code}: {error.response.reason}. {known_reason}. {doc_ref} "
-        response_error_message = stream.parse_response_error_message(error.response)
-        if response_error_message:
-            reason += response_error_message
-        return False, reason
-
-    def reasons_for_unavailable_status_codes(
-        self, stream: Stream, logger: logging.Logger, source: Optional["Source"], error: HTTPError
-    ) -> Dict[int, str]:
-        """
-        Returns a dictionary of HTTP status codes that indicate stream
-        unavailability and reasons explaining why a given status code may
-        have occurred and how the user can resolve that error, if applicable.
-
-        :param stream: stream
-        :param logger: source logger
-        :param source: optional (source)
-        :return: A dictionary of (status code, reason) where the 'reason' explains
-        why 'status code' may have occurred and how the user can resolve that
-        error, if applicable.
-        """
-        reasons_for_codes: Dict[int, str] = {
-            requests.codes.FORBIDDEN: "This is most likely due to insufficient permissions on the credentials in use. "
-            "Try to grant required permissions/scopes or re-authenticate"
-        }
-        return reasons_for_codes
-
-    @staticmethod
-    def _visit_docs_message(logger: logging.Logger, source: Optional["Source"]) -> str:
-        """
-        Creates a message indicating where to look in the documentation for
-        more information on a given source by checking the spec of that source
-        (if provided) for a 'documentationUrl'.
-
-        :param logger: source logger
-        :param source: optional (source)
-        :return: A message telling the user where to go to learn more about the source.
-        """
-        if not source:
-            return "Please visit the connector's documentation to learn more. "
-
-        try:
-            connector_spec = source.spec(logger)
-            docs_url = connector_spec.documentationUrl
-            if docs_url:
-                return f"Please visit {docs_url} to learn more. "
-            else:
-                return "Please visit the connector's documentation to learn more. "
-
-        except FileNotFoundError:  # If we are unit testing without implementing spec() method in source
-            if source:
-                docs_url = f"https://docs.airbyte.com/integrations/sources/{source.name}"
-            else:
-                docs_url = "https://docs.airbyte.com/integrations/sources/test"
-
-            return f"Please visit {docs_url} to learn more."
+        except AirbyteTracedException as error:
+            return False, error.message

airbyte_cdk/sources/streams/http/error_handlers/__init__.py

@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from .backoff_strategy import BackoffStrategy
+from .default_backoff_strategy import DefaultBackoffStrategy
+from .error_handler import ErrorHandler
+from .error_message_parser import ErrorMessageParser
+from .http_status_error_handler import HttpStatusErrorHandler
+from .json_error_message_parser import JsonErrorMessageParser
+from .response_models import ErrorResolution, ResponseAction
+
+__all__ = [
+    "BackoffStrategy",
+    "DefaultBackoffStrategy",
+    "ErrorHandler",
+    "ErrorMessageParser",
+    "HttpStatusErrorHandler",
+    "JsonErrorMessageParser",
+    "ResponseAction",
+    "ErrorResolution",
+]

airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py

@@ -0,0 +1,28 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+
+import requests
+
+
+class BackoffStrategy(ABC):
+    @abstractmethod
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int,
+    ) -> Optional[float]:
+        """
+        Override this method to dynamically determine backoff time e.g: by reading the X-Retry-After header.
+
+        This method is called only if should_backoff() returns True for the input request.
+
+        :param response_or_exception: The response or exception that caused the backoff.
+        :param attempt_count: The number of attempts already performed for this request.
+        :return how long to backoff in seconds. The return value may be a floating point number for subsecond precision. Returning None defers backoff
+        to the default backoff behavior (e.g using an exponential algorithm).
+        """
+        pass

airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+
+from typing import Optional, Union
+
+import requests
+
+from .backoff_strategy import BackoffStrategy
+
+
+class DefaultBackoffStrategy(BackoffStrategy):
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int,
+    ) -> Optional[float]:
+        return None

airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py

@@ -0,0 +1,86 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Mapping, Type, Union
+
+from requests.exceptions import InvalidSchema, InvalidURL, RequestException
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    ErrorResolution,
+    ResponseAction,
+)
+
+DEFAULT_ERROR_MAPPING: Mapping[Union[int, str, Type[Exception]], ErrorResolution] = {
+    InvalidSchema: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.config_error,
+        error_message="Invalid Protocol Schema: The endpoint that data is being requested from is using an invalid or insecure. Exception: requests.exceptions.InvalidSchema",
+    ),
+    InvalidURL: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.config_error,
+        error_message="Invalid URL specified: The endpoint that data is being requested from is not a valid URL. Exception: requests.exceptions.InvalidURL",
+    ),
+    RequestException: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="An exception occurred when making the request. Exception: requests.exceptions.RequestException",
+    ),
+    400: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.system_error,
+        error_message="Bad request. Please check your request parameters.",
+    ),
+    401: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.config_error,
+        error_message="Unauthorized. Please ensure you are authenticated correctly.",
+    ),
+    403: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.config_error,
+        error_message="Forbidden. You don't have permission to access this resource.",
+    ),
+    404: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.system_error,
+        error_message="Not found. The requested resource was not found on the server.",
+    ),
+    405: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.system_error,
+        error_message="Method not allowed. Please check your request method.",
+    ),
+    408: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Request timeout.",
+    ),
+    429: ErrorResolution(
+        response_action=ResponseAction.RATE_LIMITED,
+        failure_type=FailureType.transient_error,
+        error_message="Too many requests.",
+    ),
+    500: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Internal server error.",
+    ),
+    502: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Bad gateway.",
+    ),
+    503: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Service unavailable.",
+    ),
+    504: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Gateway timeout.",
+    ),
+}

airbyte_cdk/sources/streams/http/error_handlers/error_handler.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+
+import requests
+
+from .response_models import ErrorResolution
+
+
+class ErrorHandler(ABC):
+    """
+    Abstract base class to determine how to handle a failed HTTP request.
+    """
+
+    @property
+    @abstractmethod
+    def max_retries(self) -> Optional[int]:
+        """
+        The maximum number of retries to attempt before giving up.
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def max_time(self) -> Optional[int]:
+        """
+        The maximum amount of time in seconds to retry before giving up.
+        """
+        pass
+
+    @abstractmethod
+    def interpret_response(
+        self, response: Optional[Union[requests.Response, Exception]]
+    ) -> ErrorResolution:
+        """
+        Interpret the response or exception and return the corresponding response action, failure type, and error message.
+
+        :param response: The HTTP response object or exception raised during the request.
+        :return: A tuple containing the response action, failure type, and error message.
+        """
+        pass

airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from typing import Optional
+
+import requests
+
+
+class ErrorMessageParser(ABC):
+    @abstractmethod
+    def parse_response_error_message(self, response: requests.Response) -> Optional[str]:
+        """
+        Parse error message from response.
+        :param response: response received for the request
+        :return: error message
+        """
+        pass

airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py

@@ -0,0 +1,110 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from datetime import timedelta
+from typing import Mapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import (
+    DEFAULT_ERROR_MAPPING,
+)
+from airbyte_cdk.sources.streams.http.error_handlers.error_handler import ErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    ErrorResolution,
+    ResponseAction,
+)
+
+
+class HttpStatusErrorHandler(ErrorHandler):
+    def __init__(
+        self,
+        logger: logging.Logger,
+        error_mapping: Optional[Mapping[Union[int, str, type[Exception]], ErrorResolution]] = None,
+        max_retries: int = 5,
+        max_time: timedelta = timedelta(seconds=600),
+    ) -> None:
+        """
+        Initialize the HttpStatusErrorHandler.
+
+        :param error_mapping: Custom error mappings to extend or override the default mappings.
+        """
+        self._logger = logger
+        self._error_mapping = error_mapping or DEFAULT_ERROR_MAPPING
+        self._max_retries = max_retries
+        self._max_time = int(max_time.total_seconds())
+
+    @property
+    def max_retries(self) -> Optional[int]:
+        return self._max_retries
+
+    @property
+    def max_time(self) -> Optional[int]:
+        return self._max_time
+
+    def interpret_response(
+        self, response_or_exception: Optional[Union[requests.Response, Exception]] = None
+    ) -> ErrorResolution:
+        """
+        Interpret the response and return the corresponding response action, failure type, and error message.
+
+        :param response: The HTTP response object.
+        :return: A tuple containing the response action, failure type, and error message.
+        """
+
+        if isinstance(response_or_exception, Exception):
+            mapped_error: Optional[ErrorResolution] = self._error_mapping.get(
+                response_or_exception.__class__
+            )
+
+            if mapped_error is not None:
+                return mapped_error
+            else:
+                self._logger.error(
+                    f"Unexpected exception in error handler: {response_or_exception}"
+                )
+                return ErrorResolution(
+                    response_action=ResponseAction.RETRY,
+                    failure_type=FailureType.system_error,
+                    error_message=f"Unexpected exception in error handler: {response_or_exception}",
+                )
+
+        elif isinstance(response_or_exception, requests.Response):
+            if response_or_exception.status_code is None:
+                self._logger.error("Response does not include an HTTP status code.")
+                return ErrorResolution(
+                    response_action=ResponseAction.RETRY,
+                    failure_type=FailureType.transient_error,
+                    error_message="Response does not include an HTTP status code.",
+                )
+
+            if response_or_exception.ok:
+                return ErrorResolution(
+                    response_action=ResponseAction.SUCCESS,
+                    failure_type=None,
+                    error_message=None,
+                )
+
+            error_key = response_or_exception.status_code
+
+            mapped_error = self._error_mapping.get(error_key)
+
+            if mapped_error is not None:
+                return mapped_error
+            else:
+                self._logger.warning(f"Unexpected HTTP Status Code in error handler: '{error_key}'")
+                return ErrorResolution(
+                    response_action=ResponseAction.RETRY,
+                    failure_type=FailureType.system_error,
+                    error_message=f"Unexpected HTTP Status Code in error handler: {error_key}",
+                )
+        else:
+            self._logger.error(f"Received unexpected response type: {type(response_or_exception)}")
+            return ErrorResolution(
+                response_action=ResponseAction.FAIL,
+                failure_type=FailureType.system_error,
+                error_message=f"Received unexpected response type: {type(response_or_exception)}",
+            )

airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py

@@ -0,0 +1,52 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Optional
+
+import requests
+
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorMessageParser
+from airbyte_cdk.sources.utils.types import JsonType
+
+
+class JsonErrorMessageParser(ErrorMessageParser):
+    def _try_get_error(self, value: Optional[JsonType]) -> Optional[str]:
+        if isinstance(value, str):
+            return value
+        elif isinstance(value, list):
+            errors_in_value = [self._try_get_error(v) for v in value]
+            return ", ".join(v for v in errors_in_value if v is not None)
+        elif isinstance(value, dict):
+            new_value = (
+                value.get("message")
+                or value.get("messages")
+                or value.get("error")
+                or value.get("errors")
+                or value.get("failures")
+                or value.get("failure")
+                or value.get("detail")
+                or value.get("err")
+                or value.get("error_message")
+                or value.get("msg")
+                or value.get("reason")
+                or value.get("status_message")
+            )
+            return self._try_get_error(new_value)
+        return None
+
+    def parse_response_error_message(self, response: requests.Response) -> Optional[str]:
+        """
+        Parses the raw response object from a failed request into a user-friendly error message.
+
+        :param response:
+        :return: A user-friendly message that indicates the cause of the error
+        """
+        try:
+            body = response.json()
+            return self._try_get_error(body)
+        except requests.exceptions.JSONDecodeError:
+            try:
+                return response.content.decode("utf-8")
+            except Exception:
+                return None

airbyte_cdk/sources/streams/http/error_handlers/response_models.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional, Union
+
+import requests
+from requests import HTTPError
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
+
+
+class ResponseAction(Enum):
+    SUCCESS = "SUCCESS"
+    RETRY = "RETRY"
+    FAIL = "FAIL"
+    IGNORE = "IGNORE"
+    RATE_LIMITED = "RATE_LIMITED"
+
+
+@dataclass
+class ErrorResolution:
+    response_action: Optional[ResponseAction] = None
+    failure_type: Optional[FailureType] = None
+    error_message: Optional[str] = None
+
+
+def _format_exception_error_message(exception: Exception) -> str:
+    return f"{type(exception).__name__}: {str(exception)}"
+
+
+def _format_response_error_message(response: requests.Response) -> str:
+    try:
+        response.raise_for_status()
+    except HTTPError as exception:
+        return filter_secrets(
+            f"Response was not ok: `{str(exception)}`. Response content is: {response.text}"
+        )
+    # We purposefully do not add the response.content because the response is "ok" so there might be sensitive information in the payload.
+    # Feel free the
+    return f"Unexpected response with HTTP status {response.status_code}"
+
+
+def create_fallback_error_resolution(
+    response_or_exception: Optional[Union[requests.Response, Exception]],
+) -> ErrorResolution:
+    if response_or_exception is None:
+        # We do not expect this case to happen but if it does, it would be good to understand the cause and improve the error message
+        error_message = "Error handler did not receive a valid response or exception. This is unexpected please contact Airbyte Support"
+    elif isinstance(response_or_exception, Exception):
+        error_message = _format_exception_error_message(response_or_exception)
+    else:
+        error_message = _format_response_error_message(response_or_exception)
+
+    return ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.system_error,
+        error_message=error_message,
+    )
+
+
+SUCCESS_RESOLUTION = ErrorResolution(
+    response_action=ResponseAction.SUCCESS, failure_type=None, error_message=None
+)

airbyte_cdk/sources/streams/http/exceptions.py

@@ -3,17 +3,27 @@
 #
 
 
-from typing import Union
+from typing import Optional, Union
 
 import requests
 
 
 class BaseBackoffException(requests.exceptions.HTTPError):
-    def __init__(self, request: requests.PreparedRequest, response: requests.Response, error_message: str = ""):
-        error_message = (
-            error_message or f"Request URL: {request.url}, Response Code: {response.status_code}, Response Text: {response.text}"
-        )
-        super().__init__(error_message, request=request, response=response)
+    def __init__(
+        self,
+        request: requests.PreparedRequest,
+        response: Optional[Union[requests.Response, Exception]],
+        error_message: str = "",
+    ):
+        if isinstance(response, requests.Response):
+            error_message = (
+                error_message
+                or f"Request URL: {request.url}, Response Code: {response.status_code}, Response Text: {response.text}"
+            )
+            super().__init__(error_message, request=request, response=response)
+        else:
+            error_message = error_message or f"Request URL: {request.url}, Exception: {response}"
+            super().__init__(error_message, request=request, response=None)
 
 
 class RequestBodyException(Exception):
@@ -27,7 +37,13 @@ class UserDefinedBackoffException(BaseBackoffException):
     An exception that exposes how long it attempted to backoff
     """
 
-    def __init__(self, backoff: Union[int, float], request: requests.PreparedRequest, response: requests.Response, error_message: str = ""):
+    def __init__(
+        self,
+        backoff: Union[int, float],
+        request: requests.PreparedRequest,
+        response: Optional[Union[requests.Response, Exception]],
+        error_message: str = "",
+    ):
         """
         :param backoff: how long to backoff in seconds
         :param request: the request that triggered this backoff exception
@@ -39,3 +55,7 @@ class UserDefinedBackoffException(BaseBackoffException):
 
 class DefaultBackoffException(BaseBackoffException):
     pass
+
+
+class RateLimitBackoffException(BaseBackoffException):
+    pass