airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py

@@ -1,16 +1,23 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+import copy
+import logging
 from dataclasses import InitVar, dataclass
 from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
 
-import dpath.util
-from airbyte_cdk.models import AirbyteMessage, SyncMode, Type
+import dpath
+
+from airbyte_cdk.models import AirbyteMessage
+from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
-from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType
-from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer
-from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState
+from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
+from airbyte_cdk.sources.declarative.requesters.request_option import (
+    RequestOption,
+    RequestOptionType,
+)
+from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
+from airbyte_cdk.utils import AirbyteTracedException
 
 if TYPE_CHECKING:
     from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
@@ -24,7 +31,9 @@ class ParentStreamConfig:
     stream: The stream to read records from
     parent_key: The key of the parent stream's records that will be the stream slice key
     partition_field: The partition key
+    extra_fields: Additional field paths to include in the stream slice
     request_option: How to inject the slice value on an outgoing HTTP request
+    incremental_dependency (bool): Indicates if the parent stream should be read incrementally.
     """
 
     stream: "DeclarativeStream"  # Parent streams must be DeclarativeStream because we can't know which part of the stream slice is a partition for regular Stream
@@ -32,15 +41,27 @@ class ParentStreamConfig:
     partition_field: Union[InterpolatedString, str]
     config: Config
     parameters: InitVar[Mapping[str, Any]]
+    extra_fields: Optional[Union[List[List[str]], List[List[InterpolatedString]]]] = (
+        None  # List of field paths (arrays of strings)
+    )
     request_option: Optional[RequestOption] = None
+    incremental_dependency: bool = False
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self.parent_key = InterpolatedString.create(self.parent_key, parameters=parameters)
-        self.partition_field = InterpolatedString.create(self.partition_field, parameters=parameters)
+        self.partition_field = InterpolatedString.create(
+            self.partition_field, parameters=parameters
+        )
+        if self.extra_fields:
+            # Create InterpolatedString for each field path in extra_keys
+            self.extra_fields = [
+                [InterpolatedString.create(path, parameters=parameters) for path in key_path]
+                for key_path in self.extra_fields
+            ]
 
 
 @dataclass
-class SubstreamPartitionRouter(StreamSlicer):
+class SubstreamPartitionRouter(PartitionRouter):
     """
     Partition router that iterates over the parent's stream records and emits slices
     Will populate the state with `partition_field` and `parent_slice` so they can be accessed by other components
@@ -94,15 +115,26 @@ class SubstreamPartitionRouter(StreamSlicer):
         # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response
         return self._get_request_option(RequestOptionType.body_json, stream_slice)
 
-    def _get_request_option(self, option_type: RequestOptionType, stream_slice: Optional[StreamSlice]) -> Mapping[str, Any]:
+    def _get_request_option(
+        self, option_type: RequestOptionType, stream_slice: Optional[StreamSlice]
+    ) -> Mapping[str, Any]:
         params = {}
         if stream_slice:
             for parent_config in self.parent_stream_configs:
-                if parent_config.request_option and parent_config.request_option.inject_into == option_type:
+                if (
+                    parent_config.request_option
+                    and parent_config.request_option.inject_into == option_type
+                ):
                     key = parent_config.partition_field.eval(self.config)  # type: ignore # partition_field is always casted to an interpolated string
                     value = stream_slice.get(key)
                     if value:
-                        params.update({parent_config.request_option.field_name.eval(config=self.config): value})  # type: ignore # field_name is always casted to an interpolated string
+                        params.update(
+                            {
+                                parent_config.request_option.field_name.eval(  # type: ignore [union-attr]
+                                    config=self.config
+                                ): value
+                            }
+                        )
         return params
 
     def stream_slices(self) -> Iterable[StreamSlice]:
@@ -127,32 +159,183 @@ class SubstreamPartitionRouter(StreamSlicer):
                 parent_stream = parent_stream_config.stream
                 parent_field = parent_stream_config.parent_key.eval(self.config)  # type: ignore # parent_key is always casted to an interpolated string
                 partition_field = parent_stream_config.partition_field.eval(self.config)  # type: ignore # partition_field is always casted to an interpolated string
-                for parent_stream_slice in parent_stream.stream_slices(
-                    sync_mode=SyncMode.full_refresh, cursor_field=None, stream_state=None
-                ):
-                    empty_parent_slice = True
-                    parent_partition = parent_stream_slice.partition if parent_stream_slice else {}
-
-                    for parent_record in parent_stream.read_records(
-                        sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=parent_stream_slice, stream_state=None
-                    ):
-                        # Skip non-records (eg AirbyteLogMessage)
-                        if isinstance(parent_record, AirbyteMessage):
-                            if parent_record.type == Type.RECORD:
-                                parent_record = parent_record.record.data
-                            else:
-                                continue
-                        elif isinstance(parent_record, Record):
-                            parent_record = parent_record.data
-                        try:
-                            partition_value = dpath.util.get(parent_record, parent_field)
-                        except KeyError:
-                            pass
+                extra_fields = None
+                if parent_stream_config.extra_fields:
+                    extra_fields = [
+                        [field_path_part.eval(self.config) for field_path_part in field_path]  # type: ignore [union-attr]
+                        for field_path in parent_stream_config.extra_fields
+                    ]
+
+                # read_stateless() assumes the parent is not concurrent. This is currently okay since the concurrent CDK does
+                # not support either substreams or RFR, but something that needs to be considered once we do
+                for parent_record in parent_stream.read_only_records():
+                    parent_partition = None
+                    # Skip non-records (eg AirbyteLogMessage)
+                    if isinstance(parent_record, AirbyteMessage):
+                        self.logger.warning(
+                            f"Parent stream {parent_stream.name} returns records of type AirbyteMessage. This SubstreamPartitionRouter is not able to checkpoint incremental parent state."
+                        )
+                        if parent_record.type == MessageType.RECORD:
+                            parent_record = parent_record.record.data  # type: ignore[union-attr, assignment]  # record is always a Record
                         else:
-                            empty_parent_slice = False
-                            yield StreamSlice(
-                                partition={partition_field: partition_value, "parent_slice": parent_partition}, cursor_slice={}
-                            )
-                    # If the parent slice contains no records,
-                    if empty_parent_slice:
-                        yield from []
+                            continue
+                    elif isinstance(parent_record, Record):
+                        parent_partition = (
+                            parent_record.associated_slice.partition
+                            if parent_record.associated_slice
+                            else {}
+                        )
+                        parent_record = parent_record.data
+                    elif not isinstance(parent_record, Mapping):
+                        # The parent_record should only take the form of a Record, AirbyteMessage, or Mapping. Anything else is invalid
+                        raise AirbyteTracedException(
+                            message=f"Parent stream returned records as invalid type {type(parent_record)}"
+                        )
+                    try:
+                        partition_value = dpath.get(
+                            parent_record,  # type: ignore [arg-type]
+                            parent_field,
+                        )
+                    except KeyError:
+                        continue
+
+                    # Add extra fields
+                    extracted_extra_fields = self._extract_extra_fields(parent_record, extra_fields)
+
+                    yield StreamSlice(
+                        partition={
+                            partition_field: partition_value,
+                            "parent_slice": parent_partition or {},
+                        },
+                        cursor_slice={},
+                        extra_fields=extracted_extra_fields,
+                    )
+
+    def _extract_extra_fields(
+        self,
+        parent_record: Mapping[str, Any] | AirbyteMessage,
+        extra_fields: Optional[List[List[str]]] = None,
+    ) -> Mapping[str, Any]:
+        """
+        Extracts additional fields specified by their paths from the parent record.
+
+        Args:
+            parent_record (Mapping[str, Any]): The record from the parent stream to extract fields from.
+            extra_fields (Optional[List[List[str]]]): A list of field paths (as lists of strings) to extract from the parent record.
+
+        Returns:
+            Mapping[str, Any]: A dictionary containing the extracted fields.
+                               The keys are the joined field paths, and the values are the corresponding extracted values.
+        """
+        extracted_extra_fields = {}
+        if extra_fields:
+            for extra_field_path in extra_fields:
+                try:
+                    extra_field_value = dpath.get(
+                        parent_record,  # type: ignore [arg-type]
+                        extra_field_path,
+                    )
+                    self.logger.debug(
+                        f"Extracted extra_field_path: {extra_field_path} with value: {extra_field_value}"
+                    )
+                except KeyError:
+                    self.logger.debug(f"Failed to extract extra_field_path: {extra_field_path}")
+                    extra_field_value = None
+                extracted_extra_fields[".".join(extra_field_path)] = extra_field_value
+        return extracted_extra_fields
+
+    def set_initial_state(self, stream_state: StreamState) -> None:
+        """
+        Set the state of the parent streams.
+
+        If the `parent_state` key is missing from `stream_state`, migrate the child stream state to the parent stream's state format.
+        This migration applies only to parent streams with incremental dependencies.
+
+        Args:
+            stream_state (StreamState): The state of the streams to be set.
+
+        Example of state format:
+        {
+            "parent_state": {
+                "parent_stream_name1": {
+                    "last_updated": "2023-05-27T00:00:00Z"
+                },
+                "parent_stream_name2": {
+                    "last_updated": "2023-05-27T00:00:00Z"
+                }
+            }
+        }
+
+        Example of migrating to parent state format:
+        - Initial state:
+        {
+            "updated_at": "2023-05-27T00:00:00Z"
+        }
+        - After migration:
+        {
+            "updated_at": "2023-05-27T00:00:00Z",
+            "parent_state": {
+                "parent_stream_name": {
+                    "parent_stream_cursor": "2023-05-27T00:00:00Z"
+                }
+            }
+        }
+        """
+        if not stream_state:
+            return
+
+        parent_state = stream_state.get("parent_state", {})
+
+        # If `parent_state` doesn't exist and at least one parent stream has an incremental dependency,
+        # copy the child state to parent streams with incremental dependencies.
+        incremental_dependency = any(
+            [parent_config.incremental_dependency for parent_config in self.parent_stream_configs]
+        )
+        if not parent_state and not incremental_dependency:
+            return
+
+        if not parent_state and incremental_dependency:
+            # Attempt to retrieve child state
+            substream_state = list(stream_state.values())
+            substream_state = substream_state[0] if substream_state else {}  # type: ignore [assignment]  # Incorrect type for assignment
+            parent_state = {}
+
+            # Copy child state to parent streams with incremental dependencies
+            if substream_state:
+                for parent_config in self.parent_stream_configs:
+                    if parent_config.incremental_dependency:
+                        parent_state[parent_config.stream.name] = {
+                            parent_config.stream.cursor_field: substream_state
+                        }
+
+        # Set state for each parent stream with an incremental dependency
+        for parent_config in self.parent_stream_configs:
+            if parent_config.incremental_dependency:
+                parent_config.stream.state = parent_state.get(parent_config.stream.name, {})
+
+    def get_stream_state(self) -> Optional[Mapping[str, StreamState]]:
+        """
+        Get the state of the parent streams.
+
+        Returns:
+            StreamState: The current state of the parent streams.
+
+        Example of state format:
+        {
+            "parent_stream_name1": {
+                "last_updated": "2023-05-27T00:00:00Z"
+            },
+            "parent_stream_name2": {
+                "last_updated": "2023-05-27T00:00:00Z"
+            }
+        }
+        """
+        parent_state = {}
+        for parent_config in self.parent_stream_configs:
+            if parent_config.incremental_dependency:
+                parent_state[parent_config.stream.name] = copy.deepcopy(parent_config.stream.state)
+        return parent_state
+
+    @property
+    def logger(self) -> logging.Logger:
+        return logging.getLogger("airbyte.SubstreamPartitionRouter")

airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py

@@ -2,10 +2,24 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy
-from airbyte_cdk.sources.declarative.requesters.error_handlers.composite_error_handler import CompositeErrorHandler
-from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler
+from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import (
+    BackoffStrategy,
+)
+from airbyte_cdk.sources.declarative.requesters.error_handlers.composite_error_handler import (
+    CompositeErrorHandler,
+)
+from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import (
+    DefaultErrorHandler,
+)
 from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler
-from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import HttpResponseFilter
+from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import (
+    HttpResponseFilter,
+)
 
-__all__ = ["BackoffStrategy", "CompositeErrorHandler", "DefaultErrorHandler", "ErrorHandler", "HttpResponseFilter"]
+__all__ = [
+    "BackoffStrategy",
+    "CompositeErrorHandler",
+    "DefaultErrorHandler",
+    "ErrorHandler",
+    "HttpResponseFilter",
+]

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py

@@ -2,7 +2,9 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.constant_backoff_strategy import ConstantBackoffStrategy
+from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.constant_backoff_strategy import (
+    ConstantBackoffStrategy,
+)
 from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import (
     ExponentialBackoffStrategy,
 )

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py

@@ -6,9 +6,10 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
-from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
+from airbyte_cdk.sources.types import Config
 
 
 @dataclass
@@ -24,10 +25,21 @@ class ConstantBackoffStrategy(BackoffStrategy):
     parameters: InitVar[Mapping[str, Any]]
     config: Config
 
-    def __post_init__(self, parameters: Mapping[str, Any]):
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         if not isinstance(self.backoff_time_in_seconds, InterpolatedString):
             self.backoff_time_in_seconds = str(self.backoff_time_in_seconds)
-        self.backoff_time_in_seconds = InterpolatedString.create(self.backoff_time_in_seconds, parameters=parameters)
-
-    def backoff(self, response: requests.Response, attempt_count: int) -> Optional[float]:
-        return self.backoff_time_in_seconds.eval(self.config)
+        if isinstance(self.backoff_time_in_seconds, float):
+            self.backoff_time_in_seconds = InterpolatedString.create(
+                str(self.backoff_time_in_seconds), parameters=parameters
+            )
+        else:
+            self.backoff_time_in_seconds = InterpolatedString.create(
+                self.backoff_time_in_seconds, parameters=parameters
+            )
+
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int,
+    ) -> Optional[float]:
+        return self.backoff_time_in_seconds.eval(self.config)  # type: ignore # backoff_time_in_seconds is always cast to an interpolated string

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py

@@ -6,9 +6,10 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
-from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
+from airbyte_cdk.sources.types import Config
 
 
 @dataclass
@@ -24,10 +25,21 @@ class ExponentialBackoffStrategy(BackoffStrategy):
     config: Config
     factor: Union[float, InterpolatedString, str] = 5
 
-    def __post_init__(self, parameters: Mapping[str, Any]):
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         if not isinstance(self.factor, InterpolatedString):
             self.factor = str(self.factor)
-        self.factor = InterpolatedString.create(self.factor, parameters=parameters)
-
-    def backoff(self, response: requests.Response, attempt_count: int) -> Optional[float]:
-        return self.factor.eval(self.config) * 2**attempt_count
+        if isinstance(self.factor, float):
+            self._factor = InterpolatedString.create(str(self.factor), parameters=parameters)
+        else:
+            self._factor = InterpolatedString.create(self.factor, parameters=parameters)
+
+    @property
+    def _retry_factor(self) -> float:
+        return self._factor.eval(self.config)  # type: ignore # factor is always cast to an interpolated string
+
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int,
+    ) -> Optional[float]:
+        return self._retry_factor * 2**attempt_count  # type: ignore # factor is always cast to an interpolated string

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py

@@ -9,7 +9,9 @@ from typing import Optional
 import requests
 
 
-def get_numeric_value_from_header(response: requests.Response, header: str, regex: Optional[Pattern]) -> Optional[float]:
+def get_numeric_value_from_header(
+    response: requests.Response, header: str, regex: Optional[Pattern[str]]
+) -> Optional[float]:
     """
     Extract a header value from the response as a float
     :param response: response the extract header value from
@@ -27,7 +29,7 @@ def get_numeric_value_from_header(response: requests.Response, header: str, rege
                 header_value = match.group()
         return _as_float(header_value)
     elif isinstance(header_value, numbers.Number):
-        return float(header_value)
+        return float(header_value)  # type: ignore[arg-type]
     else:
         return None
 

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py

@@ -7,10 +7,17 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
+from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
-from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import get_numeric_value_from_header
-from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
+    get_numeric_value_from_header,
+)
+from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import (
+    BackoffStrategy,
+)
+from airbyte_cdk.sources.types import Config
+from airbyte_cdk.utils import AirbyteTracedException
 
 
 @dataclass
@@ -21,18 +28,43 @@ class WaitTimeFromHeaderBackoffStrategy(BackoffStrategy):
     Attributes:
         header (str): header to read wait time from
         regex (Optional[str]): optional regex to apply on the header to extract its value
+        max_waiting_time_in_seconds: (Optional[float]): given the value extracted from the header is greater than this value, stop the stream
     """
 
     header: Union[InterpolatedString, str]
     parameters: InitVar[Mapping[str, Any]]
     config: Config
-    regex: Optional[str] = None
+    regex: Optional[Union[InterpolatedString, str]] = None
+    max_waiting_time_in_seconds: Optional[float] = None
 
-    def __post_init__(self, parameters: Mapping[str, Any]):
-        self.regex = re.compile(self.regex) if self.regex else None
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self.regex = (
+            InterpolatedString.create(self.regex, parameters=parameters) if self.regex else None
+        )
         self.header = InterpolatedString.create(self.header, parameters=parameters)
 
-    def backoff(self, response: requests.Response, attempt_count: int) -> Optional[float]:
-        header = self.header.eval(config=self.config)
-        header_value = get_numeric_value_from_header(response, header, self.regex)
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int,
+    ) -> Optional[float]:
+        header = self.header.eval(config=self.config)  # type: ignore  # header is always cast to an interpolated stream
+        if self.regex:
+            evaled_regex = self.regex.eval(self.config)  # type: ignore # header is always cast to an interpolated string
+            regex = re.compile(evaled_regex)
+        else:
+            regex = None
+        header_value = None
+        if isinstance(response_or_exception, requests.Response):
+            header_value = get_numeric_value_from_header(response_or_exception, header, regex)
+            if (
+                self.max_waiting_time_in_seconds
+                and header_value
+                and header_value >= self.max_waiting_time_in_seconds
+            ):
+                raise AirbyteTracedException(
+                    internal_message=f"Rate limit wait time {header_value} is greater than max waiting time of {self.max_waiting_time_in_seconds} seconds. Stopping the stream...",
+                    message="The rate limit is greater than max waiting time has been reached.",
+                    failure_type=FailureType.transient_error,
+                )
         return header_value

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py

@@ -9,10 +9,15 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
-from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import get_numeric_value_from_header
-from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
+    get_numeric_value_from_header,
+)
+from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import (
+    BackoffStrategy,
+)
+from airbyte_cdk.sources.types import Config
 
 
 @dataclass
@@ -33,30 +38,40 @@ class WaitUntilTimeFromHeaderBackoffStrategy(BackoffStrategy):
     min_wait: Optional[Union[float, InterpolatedString, str]] = None
     regex: Optional[Union[InterpolatedString, str]] = None
 
-    def __post_init__(self, parameters: Mapping[str, Any]):
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self.header = InterpolatedString.create(self.header, parameters=parameters)
-        self.regex = InterpolatedString.create(self.regex, parameters=parameters) if self.regex else None
+        self.regex = (
+            InterpolatedString.create(self.regex, parameters=parameters) if self.regex else None
+        )
         if not isinstance(self.min_wait, InterpolatedString):
             self.min_wait = InterpolatedString.create(str(self.min_wait), parameters=parameters)
 
-    def backoff(self, response: requests.Response, attempt_count: int) -> Optional[float]:
+    def backoff_time(
+        self,
+        response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
+        attempt_count: int,
+    ) -> Optional[float]:
         now = time.time()
-        header = self.header.eval(self.config)
+        header = self.header.eval(self.config)  # type: ignore # header is always cast to an interpolated string
         if self.regex:
-            evaled_regex = self.regex.eval(self.config)
+            evaled_regex = self.regex.eval(self.config)  # type: ignore # header is always cast to an interpolated string
             regex = re.compile(evaled_regex)
         else:
             regex = None
-        wait_until = get_numeric_value_from_header(response, header, regex)
-        min_wait = self.min_wait.eval(self.config)
+        wait_until = None
+        if isinstance(response_or_exception, requests.Response):
+            wait_until = get_numeric_value_from_header(response_or_exception, header, regex)
+        min_wait = self.min_wait.eval(self.config)  # type: ignore # header is always cast to an interpolated string
         if wait_until is None or not wait_until:
-            return min_wait
-        if (isinstance(wait_until, str) and wait_until.isnumeric()) or isinstance(wait_until, numbers.Number):
+            return float(min_wait) if min_wait else None
+        if (isinstance(wait_until, str) and wait_until.isnumeric()) or isinstance(
+            wait_until, numbers.Number
+        ):
             wait_time = float(wait_until) - now
         else:
-            return self.min_wait
+            return float(min_wait)
         if min_wait:
-            return max(wait_time, min_wait)
+            return float(max(wait_time, min_wait))
         elif wait_time < 0:
             return None
         return wait_time

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py

@@ -2,24 +2,16 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from abc import abstractmethod
+from abc import ABC
 from dataclasses import dataclass
-from typing import Optional
 
-import requests
+from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
 
 
 @dataclass
-class BackoffStrategy:
+class DecalarativeBackoffStrategy(BackoffStrategy, ABC):
     """
+    This interface exists to retain backwards compatability with connectors that reference the declarative BackoffStrategy. As part of the effort to promote common interfaces to the Python CDK, this now extends the Python CDK backoff strategy interface.
+
     Backoff strategy defining how long to wait before retrying a request that resulted in an error.
     """
-
-    @abstractmethod
-    def backoff(self, response: requests.Response, attempt_count: int) -> Optional[float]:
-        """
-        Return time to wait before retrying the request.
-        :param response: response received for the request to retry
-        :param attempt_count: number of attempts to submit the request
-        :return: time to wait in seconds
-        """