airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

airbyte_cdk/sources/streams/core.py

@@ -1,28 +1,42 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
-
+import copy
 import inspect
+import itertools
 import logging
-import typing
 from abc import ABC, abstractmethod
-from functools import lru_cache
-from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union
+from dataclasses import dataclass
+from functools import cached_property, lru_cache
+from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union
+
+from typing_extensions import deprecated
 
 import airbyte_cdk.sources.utils.casing as casing
-from airbyte_cdk.models import AirbyteMessage, AirbyteStream, ConfiguredAirbyteStream, SyncMode
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteStream,
+    ConfiguredAirbyteStream,
+    DestinationSyncMode,
+    SyncMode,
+)
 from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.streams.checkpoint import (
+    CheckpointMode,
+    CheckpointReader,
+    Cursor,
+    CursorBasedCheckpointReader,
+    FullRefreshCheckpointReader,
+    IncrementalCheckpointReader,
+    LegacyCursorBasedCheckpointReader,
+    ResumableFullRefreshCheckpointReader,
+)
+from airbyte_cdk.sources.types import StreamSlice
 
 # list of all possible HTTP methods which can be used for sending of request bodies
 from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader
-from airbyte_cdk.sources.utils.slice_logger import SliceLogger
+from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger, SliceLogger
 from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
-from deprecated.classic import deprecated
-
-if typing.TYPE_CHECKING:
-    from airbyte_cdk.sources import Source
-    from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
 
 # A stream's read method can return one of the following types:
 # Mapping[str, Any]: The content of an AirbyteRecordMessage
@@ -31,9 +45,7 @@ StreamData = Union[Mapping[str, Any], AirbyteMessage]
 
 JsonSchema = Mapping[str, Any]
 
-# Streams that only support full refresh don't have a suitable cursor so this sentinel
-# value is used to indicate that stream should not load the incoming state value
-FULL_REFRESH_SENTINEL_STATE_KEY = "__ab_full_refresh_state_message"
+NO_CURSOR_STATE_KEY = "__ab_no_cursor_state_message"
 
 
 def package_name_from_class(cls: object) -> str:
@@ -45,10 +57,10 @@ def package_name_from_class(cls: object) -> str:
         raise ValueError(f"Could not find package name for class {cls}")
 
 
-class IncrementalMixin(ABC):
-    """Mixin to make stream incremental.
+class CheckpointMixin(ABC):
+    """Mixin for a stream that implements reading and writing the internal state used to checkpoint sync progress to the platform
 
-    class IncrementalStream(Stream, IncrementalMixin):
+    class CheckpointedStream(Stream, CheckpointMixin):
         @property
         def state(self):
             return self._state
@@ -79,11 +91,38 @@ class IncrementalMixin(ABC):
         """State setter, accept state serialized by state getter."""
 
 
+@deprecated(
+    "Deprecated as of CDK version 0.87.0. "
+    "Deprecated in favor of the `CheckpointMixin` which offers similar functionality."
+)
+class IncrementalMixin(CheckpointMixin, ABC):
+    """Mixin to make stream incremental.
+
+    class IncrementalStream(Stream, IncrementalMixin):
+        @property
+        def state(self):
+            return self._state
+
+        @state.setter
+        def state(self, value):
+            self._state[self.cursor_field] = value[self.cursor_field]
+    """
+
+
+@dataclass
+class StreamClassification:
+    is_legacy_format: bool
+    has_multiple_slices: bool
+
+
 class Stream(ABC):
     """
     Base abstract class for an Airbyte Stream. Makes no assumption of the Stream's underlying transport protocol.
     """
 
+    _configured_json_schema: Optional[Dict[str, Any]] = None
+    _exit_on_rate_limit: bool = False
+
     # Use self.logger in subclasses to log any messages
     @property
     def logger(self) -> logging.Logger:
@@ -92,7 +131,11 @@ class Stream(ABC):
     # TypeTransformer object to perform output data transformation
     transformer: TypeTransformer = TypeTransformer(TransformConfig.NoTransform)
 
-    @property
+    cursor: Optional[Cursor] = None
+
+    has_multiple_slices = False
+
+    @cached_property
     def name(self) -> str:
         """
         :return: Stream name. By default this is the implementing class name, but it can be overridden as needed.
@@ -122,62 +165,121 @@ class Stream(ABC):
     ) -> Iterable[StreamData]:
         sync_mode = configured_stream.sync_mode
         cursor_field = configured_stream.cursor_field
+        self.configured_json_schema = configured_stream.stream.json_schema
 
-        slices = self.stream_slices(
-            cursor_field=cursor_field,
-            sync_mode=sync_mode,  # todo: change this interface to no longer rely on sync_mode for behavior
-            stream_state=stream_state,
+        # WARNING: When performing a read() that uses incoming stream state, we MUST use the self.state that is defined as
+        # opposed to the incoming stream_state value. Because some connectors like ones using the file-based CDK modify
+        # state before setting the value on the Stream attribute, the most up-to-date state is derived from Stream.state
+        # instead of the stream_state parameter. This does not apply to legacy connectors using get_updated_state().
+        try:
+            stream_state = self.state  # type: ignore # we know the field might not exist...
+        except AttributeError:
+            pass
+
+        should_checkpoint = bool(state_manager)
+        checkpoint_reader = self._get_checkpoint_reader(
+            logger=logger, cursor_field=cursor_field, sync_mode=sync_mode, stream_state=stream_state
         )
-        logger.debug(f"Processing stream slices for {self.name} (sync_mode: {sync_mode.name})", extra={"stream_slices": slices})
 
-        has_slices = False
+        next_slice = checkpoint_reader.next()
         record_counter = 0
-        for _slice in slices:
-            has_slices = True
+        stream_state_tracker = copy.deepcopy(stream_state)
+        while next_slice is not None:
             if slice_logger.should_log_slice_message(logger):
-                yield slice_logger.create_slice_log_message(_slice)
+                yield slice_logger.create_slice_log_message(next_slice)
             records = self.read_records(
                 sync_mode=sync_mode,  # todo: change this interface to no longer rely on sync_mode for behavior
-                stream_slice=_slice,
+                stream_slice=next_slice,
                 stream_state=stream_state,
                 cursor_field=cursor_field or None,
             )
             for record_data_or_message in records:
                 yield record_data_or_message
                 if isinstance(record_data_or_message, Mapping) or (
-                    hasattr(record_data_or_message, "type") and record_data_or_message.type == MessageType.RECORD
+                    hasattr(record_data_or_message, "type")
+                    and record_data_or_message.type == MessageType.RECORD
                 ):
-                    record_data = record_data_or_message if isinstance(record_data_or_message, Mapping) else record_data_or_message.record
-                    stream_state = self.get_updated_state(stream_state, record_data)
+                    record_data = (
+                        record_data_or_message
+                        if isinstance(record_data_or_message, Mapping)
+                        else record_data_or_message.record
+                    )
+
+                    # Thanks I hate it. RFR fundamentally doesn't fit with the concept of the legacy Stream.get_updated_state()
+                    # method because RFR streams rely on pagination as a cursor. Stream.get_updated_state() was designed to make
+                    # the CDK manage state using specifically the last seen record. don't @ brian.lai
+                    #
+                    # Also, because the legacy incremental state case decouples observing incoming records from emitting state, it
+                    # requires that we separate CheckpointReader.observe() and CheckpointReader.get_checkpoint() which could
+                    # otherwise be combined.
+                    if self.cursor_field:
+                        # Some connectors have streams that implement get_updated_state(), but do not define a cursor_field. This
+                        # should be fixed on the stream implementation, but we should also protect against this in the CDK as well
+                        stream_state_tracker = self.get_updated_state(
+                            stream_state_tracker,
+                            record_data,  # type: ignore [arg-type]
+                        )
+                        self._observe_state(checkpoint_reader, stream_state_tracker)
                     record_counter += 1
 
-                    if sync_mode == SyncMode.incremental:
-                        # Checkpoint intervals are a bit controversial, but see below comment about why we're gating it right now
-                        checkpoint_interval = self.state_checkpoint_interval
-                        if checkpoint_interval and record_counter % checkpoint_interval == 0:
-                            airbyte_state_message = self._checkpoint_state(stream_state, state_manager)
-                            yield airbyte_state_message
+                    checkpoint_interval = self.state_checkpoint_interval
+                    checkpoint = checkpoint_reader.get_checkpoint()
+                    if (
+                        should_checkpoint
+                        and checkpoint_interval
+                        and record_counter % checkpoint_interval == 0
+                        and checkpoint is not None
+                    ):
+                        airbyte_state_message = self._checkpoint_state(
+                            checkpoint, state_manager=state_manager
+                        )
+                        yield airbyte_state_message
 
                     if internal_config.is_limit_reached(record_counter):
                         break
-
-            if sync_mode == SyncMode.incremental:
-                # Even though right now, only incremental streams running as incremental mode will emit periodic checkpoints. Rather than
-                # overhaul how refresh interacts with the platform, this positions the code so that once we want to start emitting
-                # periodic checkpoints in full refresh mode it can be done here
-                airbyte_state_message = self._checkpoint_state(stream_state, state_manager)
+            self._observe_state(checkpoint_reader)
+            checkpoint_state = checkpoint_reader.get_checkpoint()
+            if should_checkpoint and checkpoint_state is not None:
+                airbyte_state_message = self._checkpoint_state(
+                    checkpoint_state, state_manager=state_manager
+                )
                 yield airbyte_state_message
 
-        if not has_slices or sync_mode == SyncMode.full_refresh:
-            if sync_mode == SyncMode.full_refresh:
-                # We use a dummy state if there is no suitable value provided by full_refresh streams that do not have a valid cursor.
-                # Incremental streams running full_refresh mode emit a meaningful state
-                stream_state = stream_state or {FULL_REFRESH_SENTINEL_STATE_KEY: True}
+            next_slice = checkpoint_reader.next()
 
-            # We should always emit a final state message for full refresh sync or streams that do not have any slices
-            airbyte_state_message = self._checkpoint_state(stream_state, state_manager)
+        checkpoint = checkpoint_reader.get_checkpoint()
+        if should_checkpoint and checkpoint is not None:
+            airbyte_state_message = self._checkpoint_state(checkpoint, state_manager=state_manager)
             yield airbyte_state_message
 
+    def read_only_records(self, state: Optional[Mapping[str, Any]] = None) -> Iterable[StreamData]:
+        """
+        Helper method that performs a read on a stream with an optional state and emits records. If the parent stream supports
+        incremental, this operation does not update the stream's internal state (if it uses the modern state setter/getter)
+        or emit state messages.
+        """
+
+        configured_stream = ConfiguredAirbyteStream(
+            stream=AirbyteStream(
+                name=self.name,
+                json_schema={},
+                supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental],
+            ),
+            sync_mode=SyncMode.incremental if state else SyncMode.full_refresh,
+            destination_sync_mode=DestinationSyncMode.append,
+        )
+
+        yield from self.read(
+            configured_stream=configured_stream,
+            logger=self.logger,
+            slice_logger=DebugSliceLogger(),
+            stream_state=dict(state)
+            if state
+            else {},  # read() expects MutableMapping instead of Mapping which is used more often
+            state_manager=None,
+            internal_config=InternalConfig(),  # type: ignore [call-arg]
+        )
+
     @abstractmethod
     def read_records(
         self,
@@ -202,14 +304,20 @@ class Stream(ABC):
         return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema(self.name)
 
     def as_airbyte_stream(self) -> AirbyteStream:
-        stream = AirbyteStream(name=self.name, json_schema=dict(self.get_json_schema()), supported_sync_modes=[SyncMode.full_refresh])
+        stream = AirbyteStream(
+            name=self.name,
+            json_schema=dict(self.get_json_schema()),
+            supported_sync_modes=[SyncMode.full_refresh],
+            is_resumable=self.is_resumable,
+        )
 
         if self.namespace:
             stream.namespace = self.namespace
 
+        # If we can offer incremental we always should. RFR is always less reliable than incremental which uses a real cursor value
         if self.supports_incremental:
             stream.source_defined_cursor = self.source_defined_cursor
-            stream.supported_sync_modes.append(SyncMode.incremental)  # type: ignore
+            stream.supported_sync_modes.append(SyncMode.incremental)
             stream.default_cursor_field = self._wrapped_cursor_field()
 
         keys = Stream._wrapped_primary_key(self.primary_key)
@@ -225,6 +333,29 @@ class Stream(ABC):
         """
         return len(self._wrapped_cursor_field()) > 0
 
+    @property
+    def is_resumable(self) -> bool:
+        """
+        :return: True if this stream allows the checkpointing of sync progress and can resume from it on subsequent attempts.
+        This differs from supports_incremental because certain kinds of streams like those supporting resumable full refresh
+        can checkpoint progress in between attempts for improved fault tolerance. However, they will start from the beginning
+        on the next sync job.
+        """
+        if self.supports_incremental:
+            return True
+        if self.has_multiple_slices:
+            # We temporarily gate substream to not support RFR because puts a pretty high burden on connector developers
+            # to structure stream state in a very specific way. We also can't check for issubclass(HttpSubStream) because
+            # not all substreams implement the interface and it would be a circular dependency so we use parent as a surrogate
+            return False
+        elif hasattr(type(self), "state") and getattr(type(self), "state").fset is not None:
+            # Modern case where a stream manages state using getter/setter
+            return True
+        else:
+            # Legacy case where the CDK manages state via the get_updated_state() method. This is determined by checking if
+            # the stream's get_updated_state() differs from the Stream class and therefore has been overridden
+            return type(self).get_updated_state != Stream.get_updated_state
+
     def _wrapped_cursor_field(self) -> List[str]:
         return [self.cursor_field] if isinstance(self.cursor_field, str) else self.cursor_field
 
@@ -251,27 +382,15 @@ class Stream(ABC):
         """
         return True
 
-    def check_availability(self, logger: logging.Logger, source: Optional["Source"] = None) -> Tuple[bool, Optional[str]]:
-        """
-        Checks whether this stream is available.
-
-        :param logger: source logger
-        :param source: (optional) source
-        :return: A tuple of (boolean, str). If boolean is true, then this stream
-          is available, and no str is required. Otherwise, this stream is unavailable
-          for some reason and the str should describe what went wrong and how to
-          resolve the unavailability, if possible.
-        """
-        if self.availability_strategy:
-            return self.availability_strategy.check_availability(self, logger, source)
-        return True, None
-
     @property
-    def availability_strategy(self) -> Optional["AvailabilityStrategy"]:
-        """
-        :return: The AvailabilityStrategy used to check whether this stream is available.
-        """
-        return None
+    def exit_on_rate_limit(self) -> bool:
+        """Exit on rate limit getter, should return bool value. False if the stream will retry endlessly when rate limited."""
+        return self._exit_on_rate_limit
+
+    @exit_on_rate_limit.setter
+    def exit_on_rate_limit(self, value: bool) -> None:
+        """Exit on rate limit setter, accept bool value."""
+        self._exit_on_rate_limit = value
 
     @property
     @abstractmethod
@@ -282,7 +401,11 @@ class Stream(ABC):
         """
 
     def stream_slices(
-        self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None
+        self,
+        *,
+        sync_mode: SyncMode,
+        cursor_field: Optional[List[str]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[Optional[Mapping[str, Any]]]:
         """
         Override to define the slices for this stream. See the stream slicing section of the docs for more information.
@@ -292,7 +415,7 @@ class Stream(ABC):
         :param stream_state:
         :return:
         """
-        return [None]
+        yield StreamSlice(partition={}, cursor_slice={})
 
     @property
     def state_checkpoint_interval(self) -> Optional[int]:
@@ -308,11 +431,18 @@ class Stream(ABC):
         """
         return None
 
-    @deprecated(version="0.1.49", reason="You should use explicit state property instead, see IncrementalMixin docs.")
+    # Commented-out to avoid any runtime penalty, since this is used in a hot per-record codepath.
+    # To be evaluated for re-introduction here: https://github.com/airbytehq/airbyte-python-cdk/issues/116
+    # @deprecated(
+    #     "Deprecated method `get_updated_state` as of CDK version 0.1.49. "
+    #     "Please use explicit state property instead, see `IncrementalMixin` docs."
+    # )
     def get_updated_state(
         self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]
     ) -> MutableMapping[str, Any]:
-        """Override to extract state from the latest record. Needed to implement incremental sync.
+        """DEPRECATED. Please use explicit state property instead, see `IncrementalMixin` docs.
+
+        Override to extract state from the latest record. Needed to implement incremental sync.
 
         Inspects the latest record extracted from the data source and the current state object and return an updated state object.
 
@@ -325,6 +455,138 @@ class Stream(ABC):
         """
         return {}
 
+    def get_cursor(self) -> Optional[Cursor]:
+        """
+        A Cursor is an interface that a stream can implement to manage how its internal state is read and updated while
+        reading records. Historically, Python connectors had no concept of a cursor to manage state. Python streams need
+        to define a cursor implementation and override this method to manage state through a Cursor.
+        """
+        return self.cursor
+
+    def _get_checkpoint_reader(
+        self,
+        logger: logging.Logger,
+        cursor_field: Optional[List[str]],
+        sync_mode: SyncMode,
+        stream_state: MutableMapping[str, Any],
+    ) -> CheckpointReader:
+        mappings_or_slices = self.stream_slices(
+            cursor_field=cursor_field,
+            sync_mode=sync_mode,  # todo: change this interface to no longer rely on sync_mode for behavior
+            stream_state=stream_state,
+        )
+
+        # Because of poor foresight, we wrote the default Stream.stream_slices() method to return [None] which is confusing and
+        # has now normalized this behavior for connector developers. Now some connectors return [None]. This is objectively
+        # misleading and a more ideal interface is [{}] to indicate we still want to iterate over one slice, but with no
+        # specific slice values. None is bad, and now I feel bad that I have to write this hack.
+        if mappings_or_slices == [None]:
+            mappings_or_slices = [{}]
+
+        slices_iterable_copy, iterable_for_detecting_format = itertools.tee(mappings_or_slices, 2)
+        stream_classification = self._classify_stream(
+            mappings_or_slices=iterable_for_detecting_format
+        )
+
+        # Streams that override has_multiple_slices are explicitly indicating that they will iterate over
+        # multiple partitions. Inspecting slices to automatically apply the correct cursor is only needed as
+        # a backup. So if this value was already assigned to True by the stream, we don't need to reassign it
+        self.has_multiple_slices = (
+            self.has_multiple_slices or stream_classification.has_multiple_slices
+        )
+
+        cursor = self.get_cursor()
+        if cursor:
+            cursor.set_initial_state(stream_state=stream_state)
+
+        checkpoint_mode = self._checkpoint_mode
+
+        if cursor and stream_classification.is_legacy_format:
+            return LegacyCursorBasedCheckpointReader(
+                stream_slices=slices_iterable_copy, cursor=cursor, read_state_from_cursor=True
+            )
+        elif cursor:
+            return CursorBasedCheckpointReader(
+                stream_slices=slices_iterable_copy,
+                cursor=cursor,
+                read_state_from_cursor=checkpoint_mode == CheckpointMode.RESUMABLE_FULL_REFRESH,
+            )
+        elif checkpoint_mode == CheckpointMode.RESUMABLE_FULL_REFRESH:
+            # Resumable full refresh readers rely on the stream state dynamically being updated during pagination and does
+            # not iterate over a static set of slices.
+            return ResumableFullRefreshCheckpointReader(stream_state=stream_state)
+        elif checkpoint_mode == CheckpointMode.INCREMENTAL:
+            return IncrementalCheckpointReader(
+                stream_slices=slices_iterable_copy, stream_state=stream_state
+            )
+        else:
+            return FullRefreshCheckpointReader(stream_slices=slices_iterable_copy)
+
+    @property
+    def _checkpoint_mode(self) -> CheckpointMode:
+        if self.is_resumable and len(self._wrapped_cursor_field()) > 0:
+            return CheckpointMode.INCREMENTAL
+        elif self.is_resumable:
+            return CheckpointMode.RESUMABLE_FULL_REFRESH
+        else:
+            return CheckpointMode.FULL_REFRESH
+
+    @staticmethod
+    def _classify_stream(
+        mappings_or_slices: Iterator[Optional[Union[Mapping[str, Any], StreamSlice]]],
+    ) -> StreamClassification:
+        """
+        This is a bit of a crazy solution, but also the only way we can detect certain attributes about the stream since Python
+        streams do not follow consistent implementation patterns. We care about the following two attributes:
+        - is_substream: Helps to incrementally release changes since substreams w/ parents are much more complicated. Also
+          helps de-risk the release of changes that might impact all connectors
+        - uses_legacy_slice_format: Since the checkpoint reader must manage a complex state object, we opted to have it always
+          use the structured StreamSlice object. However, this requires backwards compatibility with Python sources that only
+          support the legacy mapping object
+
+        Both attributes can eventually be deprecated once stream's define this method deleted once substreams have been implemented and
+        legacy connectors all adhere to the StreamSlice object.
+        """
+        if not mappings_or_slices:
+            raise ValueError("A stream should always have at least one slice")
+        try:
+            next_slice = next(mappings_or_slices)
+            if isinstance(next_slice, StreamSlice) and next_slice == StreamSlice(
+                partition={}, cursor_slice={}
+            ):
+                is_legacy_format = False
+                slice_has_value = False
+            elif next_slice == {}:
+                is_legacy_format = True
+                slice_has_value = False
+            elif isinstance(next_slice, StreamSlice):
+                is_legacy_format = False
+                slice_has_value = True
+            else:
+                is_legacy_format = True
+                slice_has_value = True
+        except StopIteration:
+            # If the stream has no slices, the format ultimately does not matter since no data will get synced. This is technically
+            # a valid case because it is up to the stream to define its slicing behavior
+            return StreamClassification(is_legacy_format=False, has_multiple_slices=False)
+
+        if slice_has_value:
+            # If the first slice contained a partition value from the result of stream_slices(), this is a substream that might
+            # have multiple parent records to iterate over
+            return StreamClassification(
+                is_legacy_format=is_legacy_format, has_multiple_slices=slice_has_value
+            )
+
+        try:
+            # If stream_slices() returns multiple slices, this is also a substream that can potentially generate empty slices
+            next(mappings_or_slices)
+            return StreamClassification(is_legacy_format=is_legacy_format, has_multiple_slices=True)
+        except StopIteration:
+            # If the result of stream_slices() only returns a single empty stream slice, then we know this is a regular stream
+            return StreamClassification(
+                is_legacy_format=is_legacy_format, has_multiple_slices=False
+            )
+
     def log_stream_sync_configuration(self) -> None:
         """
         Logs the configuration of this stream.
@@ -338,7 +600,9 @@ class Stream(ABC):
         )
 
     @staticmethod
-    def _wrapped_primary_key(keys: Optional[Union[str, List[str], List[List[str]]]]) -> Optional[List[List[str]]]:
+    def _wrapped_primary_key(
+        keys: Optional[Union[str, List[str], List[List[str]]]],
+    ) -> Optional[List[List[str]]]:
         """
         :return: wrap the primary_key property in a list of list of strings required by the Airbyte Stream object.
         """
@@ -360,19 +624,80 @@ class Stream(ABC):
         else:
             raise ValueError(f"Element must be either list or str. Got: {type(keys)}")
 
+    def _observe_state(
+        self, checkpoint_reader: CheckpointReader, stream_state: Optional[Mapping[str, Any]] = None
+    ) -> None:
+        """
+        Convenience method that attempts to read the Stream's state using the recommended way of connector's managing their
+        own state via state setter/getter. But if we get back an AttributeError, then the legacy Stream.get_updated_state()
+        method is used as a fallback method.
+        """
+
+        # This is an inversion of the original logic that used to try state getter/setters first. As part of the work to
+        # automatically apply resumable full refresh to all streams, all HttpStream classes implement default state
+        # getter/setter methods, we should default to only using the incoming stream_state parameter value is {} which
+        # indicates the stream does not override the default get_updated_state() implementation. When the default method
+        # is not overridden, then the stream defers to self.state getter
+        if stream_state:
+            checkpoint_reader.observe(stream_state)
+        elif type(self).get_updated_state == Stream.get_updated_state:
+            # We only default to the state getter/setter if the stream does not use the legacy get_updated_state() method
+            try:
+                new_state = self.state  # type: ignore # This will always exist on HttpStreams, but may not for Stream
+                if new_state:
+                    checkpoint_reader.observe(new_state)
+            except AttributeError:
+                pass
+
     def _checkpoint_state(  # type: ignore  # ignoring typing for ConnectorStateManager because of circular dependencies
         self,
         stream_state: Mapping[str, Any],
         state_manager,
     ) -> AirbyteMessage:
-        # First attempt to retrieve the current state using the stream's state property. We receive an AttributeError if the state
-        # property is not implemented by the stream instance and as a fallback, use the stream_state retrieved from the stream
-        # instance's deprecated get_updated_state() method.
-        try:
-            state_manager.update_state_for_stream(
-                self.name, self.namespace, self.state  # type: ignore # we know the field might not exist...
+        # todo: This can be consolidated into one ConnectorStateManager.update_and_create_state_message() method, but I want
+        #  to reduce changes right now and this would span concurrent as well
+        state_manager.update_state_for_stream(self.name, self.namespace, stream_state)
+        return state_manager.create_state_message(self.name, self.namespace)  # type: ignore [no-any-return]
+
+    @property
+    def configured_json_schema(self) -> Optional[Dict[str, Any]]:
+        """
+        This property is set from the read method.
+
+        :return Optional[Dict]: JSON schema from configured catalog if provided, otherwise None.
+        """
+        return self._configured_json_schema
+
+    @configured_json_schema.setter
+    def configured_json_schema(self, json_schema: Dict[str, Any]) -> None:
+        self._configured_json_schema = self._filter_schema_invalid_properties(json_schema)
+
+    def _filter_schema_invalid_properties(
+        self, configured_catalog_json_schema: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """
+        Filters the properties in json_schema that are not present in the stream schema.
+        Configured Schemas can have very old fields, so we need to housekeeping ourselves.
+        """
+        configured_schema: Any = configured_catalog_json_schema.get("properties", {})
+        stream_schema_properties: Any = self.get_json_schema().get("properties", {})
+
+        configured_keys = configured_schema.keys()
+        stream_keys = stream_schema_properties.keys()
+        invalid_properties = configured_keys - stream_keys
+        if not invalid_properties:
+            return configured_catalog_json_schema
+
+        self.logger.warning(
+            f"Stream {self.name}: the following fields are deprecated and cannot be synced. {invalid_properties}. Refresh the connection's source schema to resolve this warning."
+        )
+
+        valid_configured_schema_properties_keys = stream_keys & configured_keys
+        valid_configured_schema_properties = {}
+
+        for configured_schema_property in valid_configured_schema_properties_keys:
+            valid_configured_schema_properties[configured_schema_property] = (
+                stream_schema_properties[configured_schema_property]
             )
 
-        except AttributeError:
-            state_manager.update_state_for_stream(self.name, self.namespace, stream_state)
-        return state_manager.create_state_message(self.name, self.namespace)
+        return {**configured_catalog_json_schema, "properties": valid_configured_schema_properties}

airbyte_cdk/sources/streams/http/__init__.py

@@ -5,5 +5,6 @@
 # Initialize Streams Package
 from .exceptions import UserDefinedBackoffException
 from .http import HttpStream, HttpSubStream
+from .http_client import HttpClient
 
-__all__ = ["HttpStream", "HttpSubStream", "UserDefinedBackoffException"]
+__all__ = ["HttpClient", "HttpStream", "HttpSubStream", "UserDefinedBackoffException"]