airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/streams/http/__init__.py

@@ -0,0 +1,10 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+# Initialize Streams Package
+from .exceptions import UserDefinedBackoffException
+from .http import HttpStream, HttpSubStream
+from .http_client import HttpClient
+
+__all__ = ["HttpClient", "HttpStream", "HttpSubStream", "UserDefinedBackoffException"]

airbyte_cdk/sources/streams/http/availability_strategy.py

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+import typing
+from typing import Optional, Tuple
+
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+
+if typing.TYPE_CHECKING:
+    from airbyte_cdk.sources import Source
+
+
+class HttpAvailabilityStrategy(AvailabilityStrategy):
+    def check_availability(
+        self, stream: Stream, logger: logging.Logger, source: Optional["Source"] = None
+    ) -> Tuple[bool, Optional[str]]:
+        """
+        Check stream availability by attempting to read the first record of the
+        stream.
+
+        :param stream: stream
+        :param logger: source logger
+        :param source: (optional) source
+        :return: A tuple of (boolean, str). If boolean is true, then the stream
+          is available, and no str is required. Otherwise, the stream is unavailable
+          for some reason and the str should describe what went wrong and how to
+          resolve the unavailability, if possible.
+        """
+        reason: Optional[str]
+        try:
+            # Some streams need a stream slice to read records (e.g. if they have a SubstreamPartitionRouter)
+            # Streams that don't need a stream slice will return `None` as their first stream slice.
+            stream_slice = self.get_first_stream_slice(stream)
+        except StopIteration:
+            # If stream_slices has no `next()` item (Note - this is different from stream_slices returning [None]!)
+            # This can happen when a substream's `stream_slices` method does a `for record in parent_records: yield <something>`
+            # without accounting for the case in which the parent stream is empty.
+            reason = f"Cannot attempt to connect to stream {stream.name} - no stream slices were found, likely because the parent stream is empty."
+            return False, reason
+        except AirbyteTracedException as error:
+            return False, error.message
+
+        try:
+            self.get_first_record_for_slice(stream, stream_slice)
+            return True, None
+        except StopIteration:
+            logger.info(f"Successfully connected to stream {stream.name}, but got 0 records.")
+            return True, None
+        except AirbyteTracedException as error:
+            return False, error.message

airbyte_cdk/sources/streams/http/error_handlers/__init__.py

@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from .backoff_strategy import BackoffStrategy
+from .default_backoff_strategy import DefaultBackoffStrategy
+from .error_handler import ErrorHandler
+from .error_message_parser import ErrorMessageParser
+from .http_status_error_handler import HttpStatusErrorHandler
+from .json_error_message_parser import JsonErrorMessageParser
+from .response_models import ErrorResolution, ResponseAction
+
+__all__ = [
+    "BackoffStrategy",
+    "DefaultBackoffStrategy",
+    "ErrorHandler",
+    "ErrorMessageParser",
+    "HttpStatusErrorHandler",
+    "JsonErrorMessageParser",
+    "ResponseAction",
+    "ErrorResolution",
+]

airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py

@@ -0,0 +1,28 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+
+import requests
+
+
+class BackoffStrategy(ABC):
+    @abstractmethod
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int,
+    ) -> Optional[float]:
+        """
+        Override this method to dynamically determine backoff time e.g: by reading the X-Retry-After header.
+
+        This method is called only if should_backoff() returns True for the input request.
+
+        :param response_or_exception: The response or exception that caused the backoff.
+        :param attempt_count: The number of attempts already performed for this request.
+        :return how long to backoff in seconds. The return value may be a floating point number for subsecond precision. Returning None defers backoff
+        to the default backoff behavior (e.g using an exponential algorithm).
+        """
+        pass

airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+
+from typing import Optional, Union
+
+import requests
+
+from .backoff_strategy import BackoffStrategy
+
+
+class DefaultBackoffStrategy(BackoffStrategy):
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int,
+    ) -> Optional[float]:
+        return None

airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py

@@ -0,0 +1,86 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Mapping, Type, Union
+
+from requests.exceptions import InvalidSchema, InvalidURL, RequestException
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    ErrorResolution,
+    ResponseAction,
+)
+
+DEFAULT_ERROR_MAPPING: Mapping[Union[int, str, Type[Exception]], ErrorResolution] = {
+    InvalidSchema: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.config_error,
+        error_message="Invalid Protocol Schema: The endpoint that data is being requested from is using an invalid or insecure. Exception: requests.exceptions.InvalidSchema",
+    ),
+    InvalidURL: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Invalid URL specified or DNS error occurred: The endpoint that data is being requested from is not a valid URL. Exception: requests.exceptions.InvalidURL",
+    ),
+    RequestException: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="An exception occurred when making the request. Exception: requests.exceptions.RequestException",
+    ),
+    400: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.system_error,
+        error_message="Bad request. Please check your request parameters.",
+    ),
+    401: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.config_error,
+        error_message="Unauthorized. Please ensure you are authenticated correctly.",
+    ),
+    403: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.config_error,
+        error_message="Forbidden. You don't have permission to access this resource.",
+    ),
+    404: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.system_error,
+        error_message="Not found. The requested resource was not found on the server.",
+    ),
+    405: ErrorResolution(
+        response_action=ResponseAction.FAIL,
+        failure_type=FailureType.system_error,
+        error_message="Method not allowed. Please check your request method.",
+    ),
+    408: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Request timeout.",
+    ),
+    429: ErrorResolution(
+        response_action=ResponseAction.RATE_LIMITED,
+        failure_type=FailureType.transient_error,
+        error_message="Too many requests.",
+    ),
+    500: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Internal server error.",
+    ),
+    502: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Bad gateway.",
+    ),
+    503: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Service unavailable.",
+    ),
+    504: ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.transient_error,
+        error_message="Gateway timeout.",
+    ),
+}

airbyte_cdk/sources/streams/http/error_handlers/error_handler.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+
+import requests
+
+from .response_models import ErrorResolution
+
+
+class ErrorHandler(ABC):
+    """
+    Abstract base class to determine how to handle a failed HTTP request.
+    """
+
+    @property
+    @abstractmethod
+    def max_retries(self) -> Optional[int]:
+        """
+        The maximum number of retries to attempt before giving up.
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def max_time(self) -> Optional[int]:
+        """
+        The maximum amount of time in seconds to retry before giving up.
+        """
+        pass
+
+    @abstractmethod
+    def interpret_response(
+        self, response: Optional[Union[requests.Response, Exception]]
+    ) -> ErrorResolution:
+        """
+        Interpret the response or exception and return the corresponding response action, failure type, and error message.
+
+        :param response: The HTTP response object or exception raised during the request.
+        :return: A tuple containing the response action, failure type, and error message.
+        """
+        pass

airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from typing import Optional
+
+import requests
+
+
+class ErrorMessageParser(ABC):
+    @abstractmethod
+    def parse_response_error_message(self, response: requests.Response) -> Optional[str]:
+        """
+        Parse error message from response.
+        :param response: response received for the request
+        :return: error message
+        """
+        pass

airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py

@@ -0,0 +1,110 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from datetime import timedelta
+from typing import Mapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import (
+    DEFAULT_ERROR_MAPPING,
+)
+from airbyte_cdk.sources.streams.http.error_handlers.error_handler import ErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    ErrorResolution,
+    ResponseAction,
+)
+
+
+class HttpStatusErrorHandler(ErrorHandler):
+    def __init__(
+        self,
+        logger: logging.Logger,
+        error_mapping: Optional[Mapping[Union[int, str, type[Exception]], ErrorResolution]] = None,
+        max_retries: int = 5,
+        max_time: timedelta = timedelta(seconds=600),
+    ) -> None:
+        """
+        Initialize the HttpStatusErrorHandler.
+
+        :param error_mapping: Custom error mappings to extend or override the default mappings.
+        """
+        self._logger = logger
+        self._error_mapping = error_mapping or DEFAULT_ERROR_MAPPING
+        self._max_retries = max_retries
+        self._max_time = int(max_time.total_seconds())
+
+    @property
+    def max_retries(self) -> Optional[int]:
+        return self._max_retries
+
+    @property
+    def max_time(self) -> Optional[int]:
+        return self._max_time
+
+    def interpret_response(
+        self, response_or_exception: Optional[Union[requests.Response, Exception]] = None
+    ) -> ErrorResolution:
+        """
+        Interpret the response and return the corresponding response action, failure type, and error message.
+
+        :param response: The HTTP response object.
+        :return: A tuple containing the response action, failure type, and error message.
+        """
+
+        if isinstance(response_or_exception, Exception):
+            mapped_error: Optional[ErrorResolution] = self._error_mapping.get(
+                response_or_exception.__class__
+            )
+
+            if mapped_error is not None:
+                return mapped_error
+            else:
+                self._logger.error(
+                    f"Unexpected exception in error handler: {response_or_exception}"
+                )
+                return ErrorResolution(
+                    response_action=ResponseAction.RETRY,
+                    failure_type=FailureType.system_error,
+                    error_message=f"Unexpected exception in error handler: {response_or_exception}",
+                )
+
+        elif isinstance(response_or_exception, requests.Response):
+            if response_or_exception.status_code is None:
+                self._logger.error("Response does not include an HTTP status code.")
+                return ErrorResolution(
+                    response_action=ResponseAction.RETRY,
+                    failure_type=FailureType.transient_error,
+                    error_message="Response does not include an HTTP status code.",
+                )
+
+            if response_or_exception.ok:
+                return ErrorResolution(
+                    response_action=ResponseAction.SUCCESS,
+                    failure_type=None,
+                    error_message=None,
+                )
+
+            error_key = response_or_exception.status_code
+
+            mapped_error = self._error_mapping.get(error_key)
+
+            if mapped_error is not None:
+                return mapped_error
+            else:
+                self._logger.warning(f"Unexpected HTTP Status Code in error handler: '{error_key}'")
+                return ErrorResolution(
+                    response_action=ResponseAction.RETRY,
+                    failure_type=FailureType.system_error,
+                    error_message=f"Unexpected HTTP Status Code in error handler: {error_key}",
+                )
+        else:
+            self._logger.error(f"Received unexpected response type: {type(response_or_exception)}")
+            return ErrorResolution(
+                response_action=ResponseAction.FAIL,
+                failure_type=FailureType.system_error,
+                error_message=f"Received unexpected response type: {type(response_or_exception)}",
+            )

airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py

@@ -0,0 +1,52 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Optional
+
+import requests
+
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorMessageParser
+from airbyte_cdk.sources.utils.types import JsonType
+
+
+class JsonErrorMessageParser(ErrorMessageParser):
+    def _try_get_error(self, value: Optional[JsonType]) -> Optional[str]:
+        if isinstance(value, str):
+            return value
+        elif isinstance(value, list):
+            errors_in_value = [self._try_get_error(v) for v in value]
+            return ", ".join(v for v in errors_in_value if v is not None)
+        elif isinstance(value, dict):
+            new_value = (
+                value.get("message")
+                or value.get("messages")
+                or value.get("error")
+                or value.get("errors")
+                or value.get("failures")
+                or value.get("failure")
+                or value.get("detail")
+                or value.get("err")
+                or value.get("error_message")
+                or value.get("msg")
+                or value.get("reason")
+                or value.get("status_message")
+            )
+            return self._try_get_error(new_value)
+        return None
+
+    def parse_response_error_message(self, response: requests.Response) -> Optional[str]:
+        """
+        Parses the raw response object from a failed request into a user-friendly error message.
+
+        :param response:
+        :return: A user-friendly message that indicates the cause of the error
+        """
+        try:
+            body = response.json()
+            return self._try_get_error(body)
+        except requests.exceptions.JSONDecodeError:
+            try:
+                return response.content.decode("utf-8")
+            except Exception:
+                return None

airbyte_cdk/sources/streams/http/error_handlers/response_models.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional, Union
+
+import requests
+from requests import HTTPError
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
+
+
+class ResponseAction(Enum):
+    SUCCESS = "SUCCESS"
+    RETRY = "RETRY"
+    FAIL = "FAIL"
+    IGNORE = "IGNORE"
+    RATE_LIMITED = "RATE_LIMITED"
+
+
+@dataclass
+class ErrorResolution:
+    response_action: Optional[ResponseAction] = None
+    failure_type: Optional[FailureType] = None
+    error_message: Optional[str] = None
+
+
+def _format_exception_error_message(exception: Exception) -> str:
+    return f"{type(exception).__name__}: {str(exception)}"
+
+
+def _format_response_error_message(response: requests.Response) -> str:
+    try:
+        response.raise_for_status()
+    except HTTPError as exception:
+        return filter_secrets(
+            f"Response was not ok: `{str(exception)}`. Response content is: {response.text}"
+        )
+    # We purposefully do not add the response.content because the response is "ok" so there might be sensitive information in the payload.
+    # Feel free the
+    return f"Unexpected response with HTTP status {response.status_code}"
+
+
+def create_fallback_error_resolution(
+    response_or_exception: Optional[Union[requests.Response, Exception]],
+) -> ErrorResolution:
+    if response_or_exception is None:
+        # We do not expect this case to happen but if it does, it would be good to understand the cause and improve the error message
+        error_message = "Error handler did not receive a valid response or exception. This is unexpected please contact Airbyte Support"
+    elif isinstance(response_or_exception, Exception):
+        error_message = _format_exception_error_message(response_or_exception)
+    else:
+        error_message = _format_response_error_message(response_or_exception)
+
+    return ErrorResolution(
+        response_action=ResponseAction.RETRY,
+        failure_type=FailureType.system_error,
+        error_message=error_message,
+    )
+
+
+SUCCESS_RESOLUTION = ErrorResolution(
+    response_action=ResponseAction.SUCCESS, failure_type=None, error_message=None
+)

airbyte_cdk/sources/streams/http/exceptions.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from typing import Optional, Union
+
+import requests
+
+
+class BaseBackoffException(requests.exceptions.HTTPError):
+    def __init__(
+        self,
+        request: requests.PreparedRequest,
+        response: Optional[Union[requests.Response, Exception]],
+        error_message: str = "",
+    ):
+        if isinstance(response, requests.Response):
+            error_message = (
+                error_message
+                or f"Request URL: {request.url}, Response Code: {response.status_code}, Response Text: {response.text}"
+            )
+            super().__init__(error_message, request=request, response=response)
+        else:
+            error_message = error_message or f"Request URL: {request.url}, Exception: {response}"
+            super().__init__(error_message, request=request, response=None)
+
+
+class RequestBodyException(Exception):
+    """
+    Raised when there are issues in configuring a request body
+    """
+
+
+class UserDefinedBackoffException(BaseBackoffException):
+    """
+    An exception that exposes how long it attempted to backoff
+    """
+
+    def __init__(
+        self,
+        backoff: Union[int, float],
+        request: requests.PreparedRequest,
+        response: Optional[Union[requests.Response, Exception]],
+        error_message: str = "",
+    ):
+        """
+        :param backoff: how long to backoff in seconds
+        :param request: the request that triggered this backoff exception
+        :param response: the response that triggered the backoff exception
+        """
+        self.backoff = backoff
+        super().__init__(request=request, response=response, error_message=error_message)
+
+
+class DefaultBackoffException(BaseBackoffException):
+    pass
+
+
+class RateLimitBackoffException(BaseBackoffException):
+    pass