airbyte-cdk 0.72.1__py3-none-any.whl → 6.13.1.dev4107__py3-none-any.whl

airbyte_cdk/sources/streams/call_rate.py

@@ -14,9 +14,8 @@ from urllib import parse
 
 import requests
 import requests_cache
-from pyrate_limiter import InMemoryBucket, Limiter
+from pyrate_limiter import InMemoryBucket, Limiter, RateItem, TimeClock
 from pyrate_limiter import Rate as PyRateRate
-from pyrate_limiter import RateItem, TimeClock
 from pyrate_limiter.exceptions import BucketFullException
 
 # prevents mypy from complaining about missing session attributes in LimiterMixin
@@ -76,7 +75,9 @@ class AbstractCallRatePolicy(abc.ABC):
         """
 
     @abc.abstractmethod
-    def update(self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]) -> None:
+    def update(
+        self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]
+    ) -> None:
         """Update call rate counting with current values
 
         :param available_calls:
@@ -202,12 +203,20 @@ class UnlimitedCallRatePolicy(BaseCallRatePolicy):
     def try_acquire(self, request: Any, weight: int) -> None:
         """Do nothing"""
 
-    def update(self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]) -> None:
+    def update(
+        self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]
+    ) -> None:
         """Do nothing"""
 
 
 class FixedWindowCallRatePolicy(BaseCallRatePolicy):
-    def __init__(self, next_reset_ts: datetime.datetime, period: timedelta, call_limit: int, matchers: list[RequestMatcher]):
+    def __init__(
+        self,
+        next_reset_ts: datetime.datetime,
+        period: timedelta,
+        call_limit: int,
+        matchers: list[RequestMatcher],
+    ):
         """A policy that allows {call_limit} calls within a {period} time interval
 
         :param next_reset_ts: next call rate reset time point
@@ -235,7 +244,8 @@ class FixedWindowCallRatePolicy(BaseCallRatePolicy):
             if self._calls_num + weight > self._call_limit:
                 reset_in = self._next_reset_ts - datetime.datetime.now()
                 error_message = (
-                    f"reached maximum number of allowed calls {self._call_limit} " f"per {self._offset} interval, next reset in {reset_in}."
+                    f"reached maximum number of allowed calls {self._call_limit} "
+                    f"per {self._offset} interval, next reset in {reset_in}."
                 )
                 raise CallRateLimitHit(
                     error=error_message,
@@ -247,7 +257,9 @@ class FixedWindowCallRatePolicy(BaseCallRatePolicy):
 
             self._calls_num += weight
 
-    def update(self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]) -> None:
+    def update(
+        self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]
+    ) -> None:
         """Update call rate counters, by default, only reacts to decreasing updates of available_calls and changes to call_reset_ts.
         We ignore updates with available_calls > current_available_calls to support call rate limits that are lower than API limits.
 
@@ -260,12 +272,18 @@ class FixedWindowCallRatePolicy(BaseCallRatePolicy):
 
             if available_calls is not None and current_available_calls > available_calls:
                 logger.debug(
-                    "got rate limit update from api, adjusting available calls from %s to %s", current_available_calls, available_calls
+                    "got rate limit update from api, adjusting available calls from %s to %s",
+                    current_available_calls,
+                    available_calls,
                 )
                 self._calls_num = self._call_limit - available_calls
 
             if call_reset_ts is not None and call_reset_ts != self._next_reset_ts:
-                logger.debug("got rate limit update from api, adjusting reset time from %s to %s", self._next_reset_ts, call_reset_ts)
+                logger.debug(
+                    "got rate limit update from api, adjusting reset time from %s to %s",
+                    self._next_reset_ts,
+                    call_reset_ts,
+                )
                 self._next_reset_ts = call_reset_ts
 
     def _update_current_window(self) -> None:
@@ -292,7 +310,10 @@ class MovingWindowCallRatePolicy(BaseCallRatePolicy):
         """
         if not rates:
             raise ValueError("The list of rates can not be empty")
-        pyrate_rates = [PyRateRate(limit=rate.limit, interval=int(rate.interval.total_seconds() * 1000)) for rate in rates]
+        pyrate_rates = [
+            PyRateRate(limit=rate.limit, interval=int(rate.interval.total_seconds() * 1000))
+            for rate in rates
+        ]
         self._bucket = InMemoryBucket(pyrate_rates)
         # Limiter will create the background task that clears old requests in the bucket
         self._limiter = Limiter(self._bucket)
@@ -320,14 +341,18 @@ class MovingWindowCallRatePolicy(BaseCallRatePolicy):
                     time_to_wait=timedelta(milliseconds=time_to_wait),
                 )
 
-    def update(self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]) -> None:
+    def update(
+        self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]
+    ) -> None:
         """Adjust call bucket to reflect the state of the API server
 
         :param available_calls:
         :param call_reset_ts:
         :return:
         """
-        if available_calls is not None and call_reset_ts is None:  # we do our best to sync buckets with API
+        if (
+            available_calls is not None and call_reset_ts is None
+        ):  # we do our best to sync buckets with API
             if available_calls == 0:
                 with self._limiter.lock:
                     items_to_add = self._bucket.count() < self._bucket.rates[0].limit
@@ -350,7 +375,9 @@ class AbstractAPIBudget(abc.ABC):
     """
 
     @abc.abstractmethod
-    def acquire_call(self, request: Any, block: bool = True, timeout: Optional[float] = None) -> None:
+    def acquire_call(
+        self, request: Any, block: bool = True, timeout: Optional[float] = None
+    ) -> None:
         """Try to get a call from budget, will block by default
 
         :param request:
@@ -375,7 +402,9 @@ class AbstractAPIBudget(abc.ABC):
 class APIBudget(AbstractAPIBudget):
     """Default APIBudget implementation"""
 
-    def __init__(self, policies: list[AbstractCallRatePolicy], maximum_attempts_to_acquire: int = 100000) -> None:
+    def __init__(
+        self, policies: list[AbstractCallRatePolicy], maximum_attempts_to_acquire: int = 100000
+    ) -> None:
         """Constructor
 
         :param policies: list of policies in this budget
@@ -392,7 +421,9 @@ class APIBudget(AbstractAPIBudget):
                 return policy
         return None
 
-    def acquire_call(self, request: Any, block: bool = True, timeout: Optional[float] = None) -> None:
+    def acquire_call(
+        self, request: Any, block: bool = True, timeout: Optional[float] = None
+    ) -> None:
         """Try to get a call from budget, will block by default.
         Matchers will be called sequentially in the same order they were added.
         The first matcher that returns True will
@@ -417,7 +448,9 @@ class APIBudget(AbstractAPIBudget):
         """
         pass
 
-    def _do_acquire(self, request: Any, policy: AbstractCallRatePolicy, block: bool, timeout: Optional[float]) -> None:
+    def _do_acquire(
+        self, request: Any, policy: AbstractCallRatePolicy, block: bool, timeout: Optional[float]
+    ) -> None:
         """Internal method to try to acquire a call credit
 
         :param request:
@@ -439,14 +472,20 @@ class APIBudget(AbstractAPIBudget):
                     else:
                         time_to_wait = exc.time_to_wait
 
-                    time_to_wait = max(timedelta(0), time_to_wait)  # sometimes we get negative duration
-                    logger.info("reached call limit %s. going to sleep for %s", exc.rate, time_to_wait)
+                    time_to_wait = max(
+                        timedelta(0), time_to_wait
+                    )  # sometimes we get negative duration
+                    logger.info(
+                        "reached call limit %s. going to sleep for %s", exc.rate, time_to_wait
+                    )
                     time.sleep(time_to_wait.total_seconds())
                 else:
                     raise
 
         if last_exception:
-            logger.info("we used all %s attempts to acquire and failed", self._maximum_attempts_to_acquire)
+            logger.info(
+                "we used all %s attempts to acquire and failed", self._maximum_attempts_to_acquire
+            )
             raise last_exception
 
 
@@ -481,9 +520,13 @@ class HttpAPIBudget(APIBudget):
             reset_ts = self.get_reset_ts_from_response(response)
             policy.update(available_calls=available_calls, call_reset_ts=reset_ts)
 
-    def get_reset_ts_from_response(self, response: requests.Response) -> Optional[datetime.datetime]:
+    def get_reset_ts_from_response(
+        self, response: requests.Response
+    ) -> Optional[datetime.datetime]:
         if response.headers.get(self._ratelimit_reset_header):
-            return datetime.datetime.fromtimestamp(int(response.headers[self._ratelimit_reset_header]))
+            return datetime.datetime.fromtimestamp(
+                int(response.headers[self._ratelimit_reset_header])
+            )
         return None
 
     def get_calls_left_from_response(self, response: requests.Response) -> Optional[int]:

airbyte_cdk/sources/streams/checkpoint/__init__.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+
+from .checkpoint_reader import (
+    CheckpointMode,
+    CheckpointReader,
+    CursorBasedCheckpointReader,
+    FullRefreshCheckpointReader,
+    IncrementalCheckpointReader,
+    LegacyCursorBasedCheckpointReader,
+    ResumableFullRefreshCheckpointReader,
+)
+from .cursor import Cursor
+from .resumable_full_refresh_cursor import ResumableFullRefreshCursor
+
+__all__ = [
+    "CheckpointMode",
+    "CheckpointReader",
+    "Cursor",
+    "CursorBasedCheckpointReader",
+    "FullRefreshCheckpointReader",
+    "IncrementalCheckpointReader",
+    "LegacyCursorBasedCheckpointReader",
+    "ResumableFullRefreshCheckpointReader",
+    "ResumableFullRefreshCursor",
+]

airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py

@@ -0,0 +1,335 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Any, Iterable, Mapping, Optional
+
+from airbyte_cdk.sources.types import StreamSlice
+
+from .cursor import Cursor
+
+
+class CheckpointMode(Enum):
+    INCREMENTAL = "incremental"
+    RESUMABLE_FULL_REFRESH = "resumable_full_refresh"
+    FULL_REFRESH = "full_refresh"
+
+
+FULL_REFRESH_COMPLETE_STATE: Mapping[str, Any] = {"__ab_full_refresh_sync_complete": True}
+
+
+class CheckpointReader(ABC):
+    """
+    CheckpointReader manages how to iterate over a stream's partitions and serves as the bridge for interpreting the current state
+    of the stream that should be emitted back to the platform.
+    """
+
+    @abstractmethod
+    def next(self) -> Optional[Mapping[str, Any]]:
+        """
+        Returns the next slice that will be used to fetch the next group of records. Returning None indicates that the reader
+        has finished iterating over all slices.
+        """
+
+    @abstractmethod
+    def observe(self, new_state: Mapping[str, Any]) -> None:
+        """
+        Updates the internal state of the checkpoint reader based on the incoming stream state from a connector.
+
+        WARNING: This is used to retain backwards compatibility with streams using the legacy get_stream_state() method.
+        In order to uptake Resumable Full Refresh, connectors must migrate streams to use the state setter/getter methods.
+        """
+
+    @abstractmethod
+    def get_checkpoint(self) -> Optional[Mapping[str, Any]]:
+        """
+        Retrieves the current state value of the stream. The connector does not emit state messages if the checkpoint value is None.
+        """
+
+
+class IncrementalCheckpointReader(CheckpointReader):
+    """
+    IncrementalCheckpointReader handles iterating through a stream based on partitioned windows of data that are determined
+    before syncing data.
+    """
+
+    def __init__(
+        self, stream_state: Mapping[str, Any], stream_slices: Iterable[Optional[Mapping[str, Any]]]
+    ):
+        self._state: Optional[Mapping[str, Any]] = stream_state
+        self._stream_slices = iter(stream_slices)
+        self._has_slices = False
+
+    def next(self) -> Optional[Mapping[str, Any]]:
+        try:
+            next_slice = next(self._stream_slices)
+            self._has_slices = True
+            return next_slice
+        except StopIteration:
+            # This is used to avoid sending a duplicate state message at the end of a sync since the stream has already
+            # emitted state at the end of each slice. If we want to avoid this extra complexity, we can also just accept
+            # that every sync emits a final duplicate state
+            if self._has_slices:
+                self._state = None
+            return None
+
+    def observe(self, new_state: Mapping[str, Any]) -> None:
+        self._state = new_state
+
+    def get_checkpoint(self) -> Optional[Mapping[str, Any]]:
+        return self._state
+
+
+class CursorBasedCheckpointReader(CheckpointReader):
+    """
+    CursorBasedCheckpointReader is used by streams that implement a Cursor in order to manage state. This allows the checkpoint
+    reader to delegate the complexity of fetching state to the cursor and focus on the iteration over a stream's partitions.
+
+    This reader supports the Cursor interface used by Python and low-code sources. Not to be confused with Cursor interface
+    that belongs to the Concurrent CDK.
+    """
+
+    def __init__(
+        self,
+        cursor: Cursor,
+        stream_slices: Iterable[Optional[Mapping[str, Any]]],
+        read_state_from_cursor: bool = False,
+    ):
+        self._cursor = cursor
+        self._stream_slices = iter(stream_slices)
+        # read_state_from_cursor is used to delineate that partitions should determine when to stop syncing dynamically according
+        # to the value of the state at runtime. This currently only applies to streams that use resumable full refresh.
+        self._read_state_from_cursor = read_state_from_cursor
+        self._current_slice: Optional[StreamSlice] = None
+        self._finished_sync = False
+        self._previous_state: Optional[Mapping[str, Any]] = None
+
+    def next(self) -> Optional[Mapping[str, Any]]:
+        try:
+            self.current_slice = self._find_next_slice()
+            return self.current_slice
+        except StopIteration:
+            self._finished_sync = True
+            return None
+
+    def observe(self, new_state: Mapping[str, Any]) -> None:
+        # Cursor based checkpoint readers don't need to observe the new state because it has already been updated by the cursor
+        # while processing records
+        pass
+
+    def get_checkpoint(self) -> Optional[Mapping[str, Any]]:
+        # This is used to avoid sending a duplicate state messages
+        new_state = self._cursor.get_stream_state()
+        if new_state != self._previous_state:
+            self._previous_state = new_state
+            return new_state
+        else:
+            return None
+
+    def _find_next_slice(self) -> StreamSlice:
+        """
+        _find_next_slice() returns the next slice of data should be synced for the current stream according to its cursor.
+        This function supports iterating over a stream's slices across two dimensions. The first dimension is the stream's
+        partitions like parent records for a substream. The inner dimension iterates over the cursor value like a date
+        range for incremental streams or a pagination checkpoint for resumable full refresh.
+
+        The basic algorithm for iterating through a stream's slices is:
+        1. The first time next() is invoked we get the first partition
+        2. If the current partition is already complete as a result of a previous sync attempt, continue iterating until
+           we find an un-synced partition.
+        2. For streams whose cursor value is determined dynamically using stream state
+            1. Get the state for the current partition
+            2. If the current partition's state is complete, continue iterating over partitions
+            3. If the current partition's state is still in progress, emit the next cursor value
+            4. If the current partition is complete as delineated by the sentinel value, get the next incomplete partition
+        3. When stream has processed all partitions, the iterator will raise a StopIteration exception signaling there are no more
+           slices left for extracting more records.
+        """
+
+        if self._read_state_from_cursor:
+            if self.current_slice is None:
+                # current_slice is None represents the first time we are iterating over a stream's slices. The first slice to
+                # sync not been assigned yet and must first be read from the iterator
+                next_slice = self.read_and_convert_slice()
+                state_for_slice = self._cursor.select_state(next_slice)
+                if state_for_slice == FULL_REFRESH_COMPLETE_STATE:
+                    # Skip every slice that already has the terminal complete value indicating that a previous attempt
+                    # successfully synced the slice
+                    has_more = True
+                    while has_more:
+                        next_slice = self.read_and_convert_slice()
+                        state_for_slice = self._cursor.select_state(next_slice)
+                        has_more = state_for_slice == FULL_REFRESH_COMPLETE_STATE
+                return StreamSlice(
+                    cursor_slice=state_for_slice or {},
+                    partition=next_slice.partition,
+                    extra_fields=next_slice.extra_fields,
+                )
+            else:
+                state_for_slice = self._cursor.select_state(self.current_slice)
+                if state_for_slice == FULL_REFRESH_COMPLETE_STATE:
+                    # If the current slice is is complete, move to the next slice and skip the next slices that already
+                    # have the terminal complete value indicating that a previous attempt was successfully read.
+                    # Dummy initialization for mypy since we'll iterate at least once to get the next slice
+                    next_candidate_slice = StreamSlice(cursor_slice={}, partition={})
+                    has_more = True
+                    while has_more:
+                        next_candidate_slice = self.read_and_convert_slice()
+                        state_for_slice = self._cursor.select_state(next_candidate_slice)
+                        has_more = state_for_slice == FULL_REFRESH_COMPLETE_STATE
+                    return StreamSlice(
+                        cursor_slice=state_for_slice or {},
+                        partition=next_candidate_slice.partition,
+                        extra_fields=next_candidate_slice.extra_fields,
+                    )
+                # The reader continues to process the current partition if it's state is still in progress
+                return StreamSlice(
+                    cursor_slice=state_for_slice or {},
+                    partition=self.current_slice.partition,
+                    extra_fields=self.current_slice.extra_fields,
+                )
+        else:
+            # Unlike RFR cursors that iterate dynamically according to how stream state is updated, most cursors operate
+            # on a fixed set of slices determined before reading records. They just iterate to the next slice
+            return self.read_and_convert_slice()
+
+    @property
+    def current_slice(self) -> Optional[StreamSlice]:
+        return self._current_slice
+
+    @current_slice.setter
+    def current_slice(self, value: StreamSlice) -> None:
+        self._current_slice = value
+
+    def read_and_convert_slice(self) -> StreamSlice:
+        next_slice = next(self._stream_slices)
+        if not isinstance(next_slice, StreamSlice):
+            raise ValueError(
+                f"{self.current_slice} should be of type StreamSlice. This is likely a bug in the CDK, please contact Airbyte support"
+            )
+        return next_slice
+
+
+class LegacyCursorBasedCheckpointReader(CursorBasedCheckpointReader):
+    """
+    This (unfortunate) class operates like an adapter to retain backwards compatibility with legacy sources that take in stream_slice
+    in the form of a Mapping instead of the StreamSlice object. Internally, the reader still operates over StreamSlices, but it
+    is instantiated with and emits stream slices in the form of a Mapping[str, Any]. The logic of how partitions and cursors
+    are iterated over is synonymous with CursorBasedCheckpointReader.
+
+    We also retain the existing top level fields defined by the connector so the fields are present on dependent methods. For example,
+    the resulting mapping structure passed back to the stream's read_records() method looks like:
+    {
+      "cursor_slice": {
+        "next_page_token": 10
+      },
+      "partition": {
+        "repository": "airbytehq/airbyte"
+      },
+      "next_page_token": 10,
+      "repository": "airbytehq/airbyte"
+    }
+    """
+
+    def __init__(
+        self,
+        cursor: Cursor,
+        stream_slices: Iterable[Optional[Mapping[str, Any]]],
+        read_state_from_cursor: bool = False,
+    ):
+        super().__init__(
+            cursor=cursor,
+            stream_slices=stream_slices,
+            read_state_from_cursor=read_state_from_cursor,
+        )
+
+    def next(self) -> Optional[Mapping[str, Any]]:
+        try:
+            self.current_slice = self._find_next_slice()
+
+            if "partition" in dict(self.current_slice):
+                raise ValueError("Stream is configured to use invalid stream slice key 'partition'")
+            elif "cursor_slice" in dict(self.current_slice):
+                raise ValueError(
+                    "Stream is configured to use invalid stream slice key 'cursor_slice'"
+                )
+
+            # We convert StreamSlice to a regular mapping because legacy connectors operate on the basic Mapping object. We
+            # also duplicate all fields at the top level for backwards compatibility for existing Python sources
+            return {
+                "partition": self.current_slice.partition,
+                "cursor_slice": self.current_slice.cursor_slice,
+                **dict(self.current_slice),
+            }
+        except StopIteration:
+            self._finished_sync = True
+            return None
+
+    def read_and_convert_slice(self) -> StreamSlice:
+        next_mapping_slice = next(self._stream_slices)
+        if not isinstance(next_mapping_slice, Mapping):
+            raise ValueError(
+                f"{self.current_slice} should be of type Mapping. This is likely a bug in the CDK, please contact Airbyte support"
+            )
+
+        # The legacy reader is instantiated with an iterable of stream slice mappings. We convert each into a StreamSlice
+        # to sanely process them during the sync and to reuse the existing Python defined cursors
+        return StreamSlice(
+            partition=next_mapping_slice,
+            cursor_slice={},
+        )
+
+
+class ResumableFullRefreshCheckpointReader(CheckpointReader):
+    """
+    ResumableFullRefreshCheckpointReader allows for iteration over an unbounded set of records based on the pagination strategy
+    of the stream. Because the number of pages is unknown, the stream's current state is used to determine whether to continue
+    fetching more pages or stopping the sync.
+    """
+
+    def __init__(self, stream_state: Mapping[str, Any]):
+        # The first attempt of an RFR stream has an empty {} incoming state, but should still make a first attempt to read records
+        # from the first page in next().
+        self._first_page = bool(stream_state == {})
+        self._state: Mapping[str, Any] = stream_state
+
+    def next(self) -> Optional[Mapping[str, Any]]:
+        if self._first_page:
+            self._first_page = False
+            return self._state
+        elif self._state == FULL_REFRESH_COMPLETE_STATE:
+            return None
+        else:
+            return self._state
+
+    def observe(self, new_state: Mapping[str, Any]) -> None:
+        self._state = new_state
+
+    def get_checkpoint(self) -> Optional[Mapping[str, Any]]:
+        return self._state or {}
+
+
+class FullRefreshCheckpointReader(CheckpointReader):
+    """
+    FullRefreshCheckpointReader iterates over data that cannot be checkpointed incrementally during the sync because the stream
+    is not capable of managing state. At the end of a sync, a final state message is emitted to signal completion.
+    """
+
+    def __init__(self, stream_slices: Iterable[Optional[Mapping[str, Any]]]):
+        self._stream_slices = iter(stream_slices)
+        self._final_checkpoint = False
+
+    def next(self) -> Optional[Mapping[str, Any]]:
+        try:
+            return next(self._stream_slices)
+        except StopIteration:
+            self._final_checkpoint = True
+            return None
+
+    def observe(self, new_state: Mapping[str, Any]) -> None:
+        pass
+
+    def get_checkpoint(self) -> Optional[Mapping[str, Any]]:
+        if self._final_checkpoint:
+            return {"__ab_no_cursor_state_message": True}
+        return None

airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py

@@ -3,16 +3,15 @@
 #
 
 from abc import ABC, abstractmethod
-from typing import Optional
+from typing import Any, Optional
 
-from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer
-from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState
+from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
 
 
-class Cursor(ABC, StreamSlicer):
+class Cursor(ABC):
     """
-    Cursors are components that allow for incremental syncs. They keep track of what data has been consumed and slices the requests based on
-    that information.
+    Cursors are components that allow for checkpointing the current state of a sync. They keep track of what data has been consumed
+    and allows for syncs to be resumed from a specific point based on that information.
     """
 
     @abstractmethod
@@ -35,17 +34,13 @@ class Cursor(ABC, StreamSlicer):
         pass
 
     @abstractmethod
-    def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None:
+    def close_slice(self, stream_slice: StreamSlice, *args: Any) -> None:
         """
-        Update state based on the stream slice and the latest record. Note that `stream_slice.cursor_slice` and
-        `most_recent_record.associated_slice` are expected to be the same but we make it explicit here that `stream_slice` should be leveraged to
-        update the state.
+        Update state based on the stream slice. Note that `stream_slice.cursor_slice` and `most_recent_record.associated_slice` are expected
+        to be the same but we make it explicit here that `stream_slice` should be leveraged to update the state. We do not pass in the
+        latest record, since cursor instances should maintain the relevant internal state on their own.
 
         :param stream_slice: slice to close
-        :param most_recent_record: the latest record we have received for the slice. This is important to consider because even if the
-          cursor emits a slice, some APIs are not able to enforce the upper boundary. The outcome is that the last_record might have a
-          higher cursor value than the slice upper boundary and if we want to reduce the duplication as much as possible, we need to
-          consider the highest value between the internal cursor, the stream slice upper boundary and the record cursor value.
         """
 
     @abstractmethod
@@ -72,3 +67,11 @@ class Cursor(ABC, StreamSlicer):
         """
         Evaluating which record is greater in terms of cursor. This is used to avoid having to capture all the records to close a slice
         """
+
+    @abstractmethod
+    def select_state(self, stream_slice: Optional[StreamSlice] = None) -> Optional[StreamState]:
+        """
+        Get the state value of a specific stream_slice. For incremental or resumable full refresh cursors which only manage state in
+        a single dimension this is the entire state object. For per-partition cursors used by substreams, this returns the state of
+        a specific parent delineated by the incoming slice's partition object.
+        """

airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+import json
+from typing import Any, Mapping
+
+
+class PerPartitionKeySerializer:
+    """
+    We are concerned of the performance of looping through the `states` list and evaluating equality on the partition. To reduce this
+    concern, we wanted to use dictionaries to map `partition -> cursor`. However, partitions are dict and dict can't be used as dict keys
+    since they are not hashable. By creating json string using the dict, we can have a use the dict as a key to the dict since strings are
+    hashable.
+    """
+
+    @staticmethod
+    def to_partition_key(to_serialize: Any) -> str:
+        # separators have changed in Python 3.4. To avoid being impacted by further change, we explicitly specify our own value
+        return json.dumps(to_serialize, indent=None, separators=(",", ":"), sort_keys=True)
+
+    @staticmethod
+    def to_partition(to_deserialize: Any) -> Mapping[str, Any]:
+        return json.loads(to_deserialize)  # type: ignore # The partition is known to be a dict, but the type hint is Any