airbyte-cdk 6.5.3rc2__py3-none-any.whl → 6.6.0__py3-none-any.whl

airbyte_cdk/sources/streams/http/http.py

@@ -14,11 +14,22 @@ from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.message.repository import InMemoryMessageRepository
 from airbyte_cdk.sources.streams.call_rate import APIBudget
 from airbyte_cdk.sources.streams.checkpoint.cursor import Cursor
-from airbyte_cdk.sources.streams.checkpoint.resumable_full_refresh_cursor import ResumableFullRefreshCursor
-from airbyte_cdk.sources.streams.checkpoint.substream_resumable_full_refresh_cursor import SubstreamResumableFullRefreshCursor
+from airbyte_cdk.sources.streams.checkpoint.resumable_full_refresh_cursor import (
+    ResumableFullRefreshCursor,
+)
+from airbyte_cdk.sources.streams.checkpoint.substream_resumable_full_refresh_cursor import (
+    SubstreamResumableFullRefreshCursor,
+)
 from airbyte_cdk.sources.streams.core import CheckpointMixin, Stream, StreamData
-from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy, ErrorHandler, HttpStatusErrorHandler
-from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction
+from airbyte_cdk.sources.streams.http.error_handlers import (
+    BackoffStrategy,
+    ErrorHandler,
+    HttpStatusErrorHandler,
+)
+from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
+    ErrorResolution,
+    ResponseAction,
+)
 from airbyte_cdk.sources.streams.http.http_client import HttpClient
 from airbyte_cdk.sources.types import Record, StreamSlice
 from airbyte_cdk.sources.utils.types import JsonType
@@ -35,9 +46,13 @@ class HttpStream(Stream, CheckpointMixin, ABC):
     """
 
     source_defined_cursor = True  # Most HTTP streams use a source defined cursor (i.e: the user can't configure it like on a SQL table)
-    page_size: Optional[int] = None  # Use this variable to define page size for API http requests with pagination support
+    page_size: Optional[int] = (
+        None  # Use this variable to define page size for API http requests with pagination support
+    )
 
-    def __init__(self, authenticator: Optional[AuthBase] = None, api_budget: Optional[APIBudget] = None):
+    def __init__(
+        self, authenticator: Optional[AuthBase] = None, api_budget: Optional[APIBudget] = None
+    ):
         self._exit_on_rate_limit: bool = False
         self._http_client = HttpClient(
             name=self.name,
@@ -55,7 +70,11 @@ class HttpStream(Stream, CheckpointMixin, ABC):
         # 2. Streams with at least one cursor_field are incremental and thus a superior sync to RFR.
         # 3. Streams overriding read_records() do not guarantee that they will call the parent implementation which can perform
         #    per-page checkpointing so RFR is only supported if a stream use the default `HttpStream.read_records()` method
-        if not self.cursor and len(self.cursor_field) == 0 and type(self).read_records is HttpStream.read_records:
+        if (
+            not self.cursor
+            and len(self.cursor_field) == 0
+            and type(self).read_records is HttpStream.read_records
+        ):
             self.cursor = ResumableFullRefreshCursor()
 
     @property
@@ -100,7 +119,10 @@ class HttpStream(Stream, CheckpointMixin, ABC):
         return "GET"
 
     @property
-    @deprecated(version="3.0.0", reason="You should set error_handler explicitly in HttpStream.get_error_handler() instead.")
+    @deprecated(
+        version="3.0.0",
+        reason="You should set error_handler explicitly in HttpStream.get_error_handler() instead.",
+    )
     def raise_on_http_errors(self) -> bool:
         """
         Override if needed. If set to False, allows opting-out of raising HTTP code exception.
@@ -108,7 +130,10 @@ class HttpStream(Stream, CheckpointMixin, ABC):
         return True
 
     @property
-    @deprecated(version="3.0.0", reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.")
+    @deprecated(
+        version="3.0.0",
+        reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
+    )
     def max_retries(self) -> Union[int, None]:
         """
         Override if needed. Specifies maximum amount of retries for backoff policy. Return None for no limit.
@@ -116,7 +141,10 @@ class HttpStream(Stream, CheckpointMixin, ABC):
         return 5
 
     @property
-    @deprecated(version="3.0.0", reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.")
+    @deprecated(
+        version="3.0.0",
+        reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
+    )
     def max_time(self) -> Union[int, None]:
         """
         Override if needed. Specifies maximum total waiting time (in seconds) for backoff policy. Return None for no limit.
@@ -124,7 +152,10 @@ class HttpStream(Stream, CheckpointMixin, ABC):
         return 60 * 10
 
     @property
-    @deprecated(version="3.0.0", reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.")
+    @deprecated(
+        version="3.0.0",
+        reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
+    )
     def retry_factor(self) -> float:
         """
         Override if needed. Specifies factor for backoff policy.
@@ -262,7 +293,10 @@ class HttpStream(Stream, CheckpointMixin, ABC):
         """
         if hasattr(self, "should_retry"):
             error_handler = HttpStreamAdapterHttpStatusErrorHandler(
-                stream=self, logger=logging.getLogger(), max_retries=self.max_retries, max_time=timedelta(seconds=self.max_time or 0)
+                stream=self,
+                logger=logging.getLogger(),
+                max_retries=self.max_retries,
+                max_time=timedelta(seconds=self.max_time or 0),
             )
             return error_handler
         else:
@@ -333,13 +367,17 @@ class HttpStream(Stream, CheckpointMixin, ABC):
         # A cursor_field indicates this is an incremental stream which offers better checkpointing than RFR enabled via the cursor
         if self.cursor_field or not isinstance(self.get_cursor(), ResumableFullRefreshCursor):
             yield from self._read_pages(
-                lambda req, res, state, _slice: self.parse_response(res, stream_slice=_slice, stream_state=state),
+                lambda req, res, state, _slice: self.parse_response(
+                    res, stream_slice=_slice, stream_state=state
+                ),
                 stream_slice,
                 stream_state,
             )
         else:
             yield from self._read_single_page(
-                lambda req, res, state, _slice: self.parse_response(res, stream_slice=_slice, stream_state=state),
+                lambda req, res, state, _slice: self.parse_response(
+                    res, stream_slice=_slice, stream_state=state
+                ),
                 stream_slice,
                 stream_state,
             )
@@ -373,7 +411,13 @@ class HttpStream(Stream, CheckpointMixin, ABC):
     def _read_pages(
         self,
         records_generator_fn: Callable[
-            [requests.PreparedRequest, requests.Response, Mapping[str, Any], Optional[Mapping[str, Any]]], Iterable[StreamData]
+            [
+                requests.PreparedRequest,
+                requests.Response,
+                Mapping[str, Any],
+                Optional[Mapping[str, Any]],
+            ],
+            Iterable[StreamData],
         ],
         stream_slice: Optional[Mapping[str, Any]] = None,
         stream_state: Optional[Mapping[str, Any]] = None,
@@ -403,19 +447,29 @@ class HttpStream(Stream, CheckpointMixin, ABC):
     def _read_single_page(
         self,
         records_generator_fn: Callable[
-            [requests.PreparedRequest, requests.Response, Mapping[str, Any], Optional[Mapping[str, Any]]], Iterable[StreamData]
+            [
+                requests.PreparedRequest,
+                requests.Response,
+                Mapping[str, Any],
+                Optional[Mapping[str, Any]],
+            ],
+            Iterable[StreamData],
         ],
         stream_slice: Optional[Mapping[str, Any]] = None,
         stream_state: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[StreamData]:
-        partition, cursor_slice, remaining_slice = self._extract_slice_fields(stream_slice=stream_slice)
+        partition, cursor_slice, remaining_slice = self._extract_slice_fields(
+            stream_slice=stream_slice
+        )
         stream_state = stream_state or {}
         next_page_token = cursor_slice or None
 
         request, response = self._fetch_next_page(remaining_slice, stream_state, next_page_token)
         yield from records_generator_fn(request, response, stream_state, remaining_slice)
 
-        next_page_token = self.next_page_token(response) or {"__ab_full_refresh_sync_complete": True}
+        next_page_token = self.next_page_token(response) or {
+            "__ab_full_refresh_sync_complete": True
+        }
 
         cursor = self.get_cursor()
         if cursor:
@@ -425,7 +479,9 @@ class HttpStream(Stream, CheckpointMixin, ABC):
         yield from []
 
     @staticmethod
-    def _extract_slice_fields(stream_slice: Optional[Mapping[str, Any]]) -> tuple[Mapping[str, Any], Mapping[str, Any], Mapping[str, Any]]:
+    def _extract_slice_fields(
+        stream_slice: Optional[Mapping[str, Any]],
+    ) -> tuple[Mapping[str, Any], Mapping[str, Any], Mapping[str, Any]]:
         if not stream_slice:
             return {}, {}, {}
 
@@ -439,7 +495,11 @@ class HttpStream(Stream, CheckpointMixin, ABC):
             # fields for the partition and cursor_slice value
             partition = stream_slice.get("partition", {})
             cursor_slice = stream_slice.get("cursor_slice", {})
-            remaining = {key: val for key, val in stream_slice.items() if key != "partition" and key != "cursor_slice"}
+            remaining = {
+                key: val
+                for key, val in stream_slice.items()
+                if key != "partition" and key != "cursor_slice"
+            }
         return partition, cursor_slice, remaining
 
     def _fetch_next_page(
@@ -448,18 +508,41 @@ class HttpStream(Stream, CheckpointMixin, ABC):
         stream_state: Optional[Mapping[str, Any]] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Tuple[requests.PreparedRequest, requests.Response]:
-
         request, response = self._http_client.send_request(
             http_method=self.http_method,
             url=self._join_url(
                 self.url_base,
-                self.path(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
+                self.path(
+                    stream_state=stream_state,
+                    stream_slice=stream_slice,
+                    next_page_token=next_page_token,
+                ),
+            ),
+            request_kwargs=self.request_kwargs(
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
+            ),
+            headers=self.request_headers(
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
+            ),
+            params=self.request_params(
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
+            ),
+            json=self.request_body_json(
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
+            ),
+            data=self.request_body_data(
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
             ),
-            request_kwargs=self.request_kwargs(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
-            headers=self.request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
-            params=self.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
-            json=self.request_body_json(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
-            data=self.request_body_data(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
             dedupe_query_params=True,
             log_formatter=self.get_log_formatter(),
             exit_on_rate_limit=self.exit_on_rate_limit,
@@ -482,18 +565,27 @@ class HttpSubStream(HttpStream, ABC):
         """
         super().__init__(**kwargs)
         self.parent = parent
-        self.has_multiple_slices = True  # Substreams are based on parent records which implies there are multiple slices
+        self.has_multiple_slices = (
+            True  # Substreams are based on parent records which implies there are multiple slices
+        )
 
         # There are three conditions that dictate if RFR should automatically be applied to a stream
         # 1. Streams that explicitly initialize their own cursor should defer to it and not automatically apply RFR
         # 2. Streams with at least one cursor_field are incremental and thus a superior sync to RFR.
         # 3. Streams overriding read_records() do not guarantee that they will call the parent implementation which can perform
         #    per-page checkpointing so RFR is only supported if a stream use the default `HttpStream.read_records()` method
-        if not self.cursor and len(self.cursor_field) == 0 and type(self).read_records is HttpStream.read_records:
+        if (
+            not self.cursor
+            and len(self.cursor_field) == 0
+            and type(self).read_records is HttpStream.read_records
+        ):
             self.cursor = SubstreamResumableFullRefreshCursor()
 
     def stream_slices(
-        self, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None
+        self,
+        sync_mode: SyncMode,
+        cursor_field: Optional[List[str]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[Optional[Mapping[str, Any]]]:
         # read_stateless() assumes the parent is not concurrent. This is currently okay since the concurrent CDK does
         # not support either substreams or RFR, but something that needs to be considered once we do
@@ -509,7 +601,10 @@ class HttpSubStream(HttpStream, ABC):
             yield {"parent": parent_record}
 
 
-@deprecated(version="3.0.0", reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.")
+@deprecated(
+    version="3.0.0",
+    reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
+)
 class HttpStreamAdapterBackoffStrategy(BackoffStrategy):
     def __init__(self, stream: HttpStream):
         self.stream = stream
@@ -522,13 +617,18 @@ class HttpStreamAdapterBackoffStrategy(BackoffStrategy):
         return self.stream.backoff_time(response_or_exception)  # type: ignore # noqa  # HttpStream.backoff_time has been deprecated
 
 
-@deprecated(version="3.0.0", reason="You should set error_handler explicitly in HttpStream.get_error_handler() instead.")
+@deprecated(
+    version="3.0.0",
+    reason="You should set error_handler explicitly in HttpStream.get_error_handler() instead.",
+)
 class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
     def __init__(self, stream: HttpStream, **kwargs):  # type: ignore # noqa
         self.stream = stream
         super().__init__(**kwargs)
 
-    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
+    def interpret_response(
+        self, response_or_exception: Optional[Union[requests.Response, Exception]] = None
+    ) -> ErrorResolution:
         if isinstance(response_or_exception, Exception):
             return super().interpret_response(response_or_exception)
         elif isinstance(response_or_exception, requests.Response):

airbyte_cdk/sources/streams/http/http_client.py

@@ -44,7 +44,9 @@ from airbyte_cdk.sources.streams.http.rate_limiting import (
     user_defined_backoff_handler,
 )
 from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
-from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message
+from airbyte_cdk.utils.stream_status_utils import (
+    as_airbyte_message as stream_status_as_airbyte_message,
+)
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 from requests.auth import AuthBase
 
@@ -69,7 +71,6 @@ class MessageRepresentationAirbyteTracedErrors(AirbyteTracedException):
 
 
 class HttpClient:
-
     _DEFAULT_MAX_RETRY: int = 5
     _DEFAULT_MAX_TIME: int = 60 * 10
 
@@ -95,7 +96,10 @@ class HttpClient:
             self._use_cache = use_cache
             self._session = self._request_session()
             self._session.mount(
-                "https://", requests.adapters.HTTPAdapter(pool_connections=MAX_CONNECTION_POOL_SIZE, pool_maxsize=MAX_CONNECTION_POOL_SIZE)
+                "https://",
+                requests.adapters.HTTPAdapter(
+                    pool_connections=MAX_CONNECTION_POOL_SIZE, pool_maxsize=MAX_CONNECTION_POOL_SIZE
+                ),
             )
         if isinstance(authenticator, AuthBase):
             self._session.auth = authenticator
@@ -134,7 +138,9 @@ class HttpClient:
                 sqlite_path = str(Path(cache_dir) / self.cache_filename)
             else:
                 sqlite_path = "file::memory:?cache=shared"
-            return CachedLimiterSession(sqlite_path, backend="sqlite", api_budget=self._api_budget, match_headers=True)  # type: ignore # there are no typeshed stubs for requests_cache
+            return CachedLimiterSession(
+                sqlite_path, backend="sqlite", api_budget=self._api_budget, match_headers=True
+            )  # type: ignore # there are no typeshed stubs for requests_cache
         else:
             return LimiterSession(api_budget=self._api_budget)
 
@@ -145,7 +151,9 @@ class HttpClient:
         if isinstance(self._session, requests_cache.CachedSession):
             self._session.cache.clear()  # type: ignore # cache.clear is not typed
 
-    def _dedupe_query_params(self, url: str, params: Optional[Mapping[str, str]]) -> Mapping[str, str]:
+    def _dedupe_query_params(
+        self, url: str, params: Optional[Mapping[str, str]]
+    ) -> Mapping[str, str]:
         """
         Remove query parameters from params mapping if they are already encoded in the URL.
         :param url: URL with
@@ -157,7 +165,9 @@ class HttpClient:
         query_string = urllib.parse.urlparse(url).query
         query_dict = {k: v[0] for k, v in urllib.parse.parse_qs(query_string).items()}
 
-        duplicate_keys_with_same_value = {k for k in query_dict.keys() if str(params.get(k)) == str(query_dict[k])}
+        duplicate_keys_with_same_value = {
+            k for k in query_dict.keys() if str(params.get(k)) == str(query_dict[k])
+        }
         return {k: v for k, v in params.items() if k not in duplicate_keys_with_same_value}
 
     def _create_prepared_request(
@@ -184,7 +194,9 @@ class HttpClient:
                 args["json"] = json
             elif data:
                 args["data"] = data
-        prepared_request: requests.PreparedRequest = self._session.prepare_request(requests.Request(**args))
+        prepared_request: requests.PreparedRequest = self._session.prepare_request(
+            requests.Request(**args)
+        )
 
         return prepared_request
 
@@ -205,7 +217,11 @@ class HttpClient:
         """
         Determines the max time based on the provided error handler.
         """
-        return self._error_handler.max_time if self._error_handler.max_time is not None else self._DEFAULT_MAX_TIME
+        return (
+            self._error_handler.max_time
+            if self._error_handler.max_time is not None
+            else self._DEFAULT_MAX_TIME
+        )
 
     def _send_with_retry(
         self,
@@ -229,11 +245,20 @@ class HttpClient:
         max_tries = max(0, max_retries) + 1
         max_time = self._max_time
 
-        user_backoff_handler = user_defined_backoff_handler(max_tries=max_tries, max_time=max_time)(self._send)
+        user_backoff_handler = user_defined_backoff_handler(max_tries=max_tries, max_time=max_time)(
+            self._send
+        )
         rate_limit_backoff_handler = rate_limit_default_backoff_handler()
-        backoff_handler = http_client_default_backoff_handler(max_tries=max_tries, max_time=max_time)
+        backoff_handler = http_client_default_backoff_handler(
+            max_tries=max_tries, max_time=max_time
+        )
         # backoff handlers wrap _send, so it will always return a response
-        response = backoff_handler(rate_limit_backoff_handler(user_backoff_handler))(request, request_kwargs, log_formatter=log_formatter, exit_on_rate_limit=exit_on_rate_limit)  # type: ignore # mypy can't infer that backoff_handler wraps _send
+        response = backoff_handler(rate_limit_backoff_handler(user_backoff_handler))(
+            request,
+            request_kwargs,
+            log_formatter=log_formatter,
+            exit_on_rate_limit=exit_on_rate_limit,
+        )  # type: ignore # mypy can't infer that backoff_handler wraps _send
 
         return response
 
@@ -244,7 +269,6 @@ class HttpClient:
         log_formatter: Optional[Callable[[requests.Response], Any]] = None,
         exit_on_rate_limit: Optional[bool] = False,
     ) -> requests.Response:
-
         if request not in self._request_attempt_count:
             self._request_attempt_count[request] = 1
         else:
@@ -253,7 +277,8 @@ class HttpClient:
                 self._session.auth(request)
 
         self._logger.debug(
-            "Making outbound API request", extra={"headers": request.headers, "url": request.url, "request_body": request.body}
+            "Making outbound API request",
+            extra={"headers": request.headers, "url": request.url, "request_body": request.body},
         )
 
         response: Optional[requests.Response] = None
@@ -264,7 +289,9 @@ class HttpClient:
         except requests.RequestException as e:
             exc = e
 
-        error_resolution: ErrorResolution = self._error_handler.interpret_response(response if response is not None else exc)
+        error_resolution: ErrorResolution = self._error_handler.interpret_response(
+            response if response is not None else exc
+        )
 
         # Evaluation of response.text can be heavy, for example, if streaming a large response
         # Do it only in debug mode
@@ -276,11 +303,20 @@ class HttpClient:
                 )
             else:
                 self._logger.debug(
-                    "Receiving response", extra={"headers": response.headers, "status": response.status_code, "body": response.text}
+                    "Receiving response",
+                    extra={
+                        "headers": response.headers,
+                        "status": response.status_code,
+                        "body": response.text,
+                    },
                 )
 
         # Request/response logging for declarative cdk
-        if log_formatter is not None and response is not None and self._message_repository is not None:
+        if (
+            log_formatter is not None
+            and response is not None
+            and self._message_repository is not None
+        ):
             formatter = log_formatter
             self._message_repository.log_message(
                 Level.DEBUG,
@@ -288,7 +324,11 @@ class HttpClient:
             )
 
         self._handle_error_resolution(
-            response=response, exc=exc, request=request, error_resolution=error_resolution, exit_on_rate_limit=exit_on_rate_limit
+            response=response,
+            exc=exc,
+            request=request,
+            error_resolution=error_resolution,
+            exit_on_rate_limit=exit_on_rate_limit,
         )
 
         return response  # type: ignore # will either return a valid response of type requests.Response or raise an exception
@@ -307,7 +347,9 @@ class HttpClient:
             reasons = [AirbyteStreamStatusReason(type=AirbyteStreamStatusReasonType.RATE_LIMITED)]
             message = orjson.dumps(
                 AirbyteMessageSerializer.dump(
-                    stream_status_as_airbyte_message(StreamDescriptor(name=self._name), AirbyteStreamStatus.RUNNING, reasons)
+                    stream_status_as_airbyte_message(
+                        StreamDescriptor(name=self._name), AirbyteStreamStatus.RUNNING, reasons
+                    )
                 )
             ).decode()
 
@@ -321,7 +363,9 @@ class HttpClient:
             if response is not None:
                 error_message = f"'{request.method}' request to '{request.url}' failed with status code '{response.status_code}' and error message '{self._error_message_parser.parse_response_error_message(response)}'"
             else:
-                error_message = f"'{request.method}' request to '{request.url}' failed with exception: '{exc}'"
+                error_message = (
+                    f"'{request.method}' request to '{request.url}' failed with exception: '{exc}'"
+                )
 
             raise MessageRepresentationAirbyteTracedErrors(
                 internal_message=error_message,
@@ -331,20 +375,22 @@ class HttpClient:
 
         elif error_resolution.response_action == ResponseAction.IGNORE:
             if response is not None:
-                log_message = (
-                    f"Ignoring response for '{request.method}' request to '{request.url}' with response code '{response.status_code}'"
-                )
+                log_message = f"Ignoring response for '{request.method}' request to '{request.url}' with response code '{response.status_code}'"
             else:
                 log_message = f"Ignoring response for '{request.method}' request to '{request.url}' with error '{exc}'"
 
             self._logger.info(error_resolution.error_message or log_message)
 
         # TODO: Consider dynamic retry count depending on subsequent error codes
-        elif error_resolution.response_action == ResponseAction.RETRY or error_resolution.response_action == ResponseAction.RATE_LIMITED:
+        elif (
+            error_resolution.response_action == ResponseAction.RETRY
+            or error_resolution.response_action == ResponseAction.RATE_LIMITED
+        ):
             user_defined_backoff_time = None
             for backoff_strategy in self._backoff_strategies:
                 backoff_time = backoff_strategy.backoff_time(
-                    response_or_exception=response if response is not None else exc, attempt_count=self._request_attempt_count[request]
+                    response_or_exception=response if response is not None else exc,
+                    attempt_count=self._request_attempt_count[request],
                 )
                 if backoff_time:
                     user_defined_backoff_time = backoff_time
@@ -354,7 +400,10 @@ class HttpClient:
                 or f"Request to {request.url} failed with failure type {error_resolution.failure_type}, response action {error_resolution.response_action}."
             )
 
-            retry_endlessly = error_resolution.response_action == ResponseAction.RATE_LIMITED and not exit_on_rate_limit
+            retry_endlessly = (
+                error_resolution.response_action == ResponseAction.RATE_LIMITED
+                and not exit_on_rate_limit
+            )
 
             if user_defined_backoff_time:
                 raise UserDefinedBackoffException(
@@ -365,10 +414,14 @@ class HttpClient:
                 )
 
             elif retry_endlessly:
-                raise RateLimitBackoffException(request=request, response=response or exc, error_message=error_message)
+                raise RateLimitBackoffException(
+                    request=request, response=response or exc, error_message=error_message
+                )
 
             raise DefaultBackoffException(
-                request=request, response=(response if response is not None else exc), error_message=error_message
+                request=request,
+                response=(response if response is not None else exc),
+                error_message=error_message,
             )
 
         elif response:
@@ -400,11 +453,20 @@ class HttpClient:
         """
 
         request: requests.PreparedRequest = self._create_prepared_request(
-            http_method=http_method, url=url, dedupe_query_params=dedupe_query_params, headers=headers, params=params, json=json, data=data
+            http_method=http_method,
+            url=url,
+            dedupe_query_params=dedupe_query_params,
+            headers=headers,
+            params=params,
+            json=json,
+            data=data,
         )
 
         response: requests.Response = self._send_with_retry(
-            request=request, request_kwargs=request_kwargs, log_formatter=log_formatter, exit_on_rate_limit=exit_on_rate_limit
+            request=request,
+            request_kwargs=request_kwargs,
+            log_formatter=log_formatter,
+            exit_on_rate_limit=exit_on_rate_limit,
         )
 
         return request, response

airbyte_cdk/sources/streams/http/rate_limiting.py

@@ -10,7 +10,11 @@ from typing import Any, Callable, Mapping, Optional
 import backoff
 from requests import PreparedRequest, RequestException, Response, codes, exceptions
 
-from .exceptions import DefaultBackoffException, RateLimitBackoffException, UserDefinedBackoffException
+from .exceptions import (
+    DefaultBackoffException,
+    RateLimitBackoffException,
+    UserDefinedBackoffException,
+)
 
 TRANSIENT_EXCEPTIONS = (
     DefaultBackoffException,
@@ -32,7 +36,9 @@ def default_backoff_handler(
     def log_retry_attempt(details: Mapping[str, Any]) -> None:
         _, exc, _ = sys.exc_info()
         if isinstance(exc, RequestException) and exc.response:
-            logger.info(f"Status code: {exc.response.status_code!r}, Response Content: {exc.response.content!r}")
+            logger.info(
+                f"Status code: {exc.response.status_code!r}, Response Content: {exc.response.content!r}"
+            )
         logger.info(
             f"Caught retryable error '{str(exc)}' after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..."
         )
@@ -71,7 +77,9 @@ def http_client_default_backoff_handler(
     def log_retry_attempt(details: Mapping[str, Any]) -> None:
         _, exc, _ = sys.exc_info()
         if isinstance(exc, RequestException) and exc.response:
-            logger.info(f"Status code: {exc.response.status_code!r}, Response Content: {exc.response.content!r}")
+            logger.info(
+                f"Status code: {exc.response.status_code!r}, Response Content: {exc.response.content!r}"
+            )
         logger.info(
             f"Caught retryable error '{str(exc)}' after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..."
         )
@@ -99,7 +107,9 @@ def user_defined_backoff_handler(
         _, exc, _ = sys.exc_info()
         if isinstance(exc, UserDefinedBackoffException):
             if exc.response:
-                logger.info(f"Status code: {exc.response.status_code!r}, Response Content: {exc.response.content!r}")
+                logger.info(
+                    f"Status code: {exc.response.status_code!r}, Response Content: {exc.response.content!r}"
+                )
             retry_after = exc.backoff
             logger.info(f"Retrying. Sleeping for {retry_after} seconds")
             time.sleep(retry_after + 1)  # extra second to cover any fractions of second
@@ -107,7 +117,9 @@ def user_defined_backoff_handler(
     def log_give_up(details: Mapping[str, Any]) -> None:
         _, exc, _ = sys.exc_info()
         if isinstance(exc, RequestException):
-            logger.error(f"Max retry limit reached in {details['elapsed']}s. Request: {exc.request}, Response: {exc.response}")
+            logger.error(
+                f"Max retry limit reached in {details['elapsed']}s. Request: {exc.request}, Response: {exc.response}"
+            )
         else:
             logger.error("Max retry limit reached for unknown request and response")
 
@@ -124,11 +136,15 @@ def user_defined_backoff_handler(
     )
 
 
-def rate_limit_default_backoff_handler(**kwargs: Any) -> Callable[[SendRequestCallableType], SendRequestCallableType]:
+def rate_limit_default_backoff_handler(
+    **kwargs: Any,
+) -> Callable[[SendRequestCallableType], SendRequestCallableType]:
     def log_retry_attempt(details: Mapping[str, Any]) -> None:
         _, exc, _ = sys.exc_info()
         if isinstance(exc, RequestException) and exc.response:
-            logger.info(f"Status code: {exc.response.status_code!r}, Response Content: {exc.response.content!r}")
+            logger.info(
+                f"Status code: {exc.response.status_code!r}, Response Content: {exc.response.content!r}"
+            )
         logger.info(
             f"Caught retryable error '{str(exc)}' after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..."
         )