airbyte-cdk 7.3.2__tar.gz → 7.3.3__tar.gz

{airbyte_cdk-7.3.2 → airbyte_cdk-7.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 7.3.2
+Version: 7.3.3
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-7.3.2 → airbyte_cdk-7.3.3}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -2279,12 +2279,14 @@ definitions:
           - FAIL
           - RETRY
           - IGNORE
+          - RESET_PAGINATION
           - RATE_LIMITED
         examples:
           - SUCCESS
           - FAIL
           - RETRY
           - IGNORE
+          - RESET_PAGINATION
           - RATE_LIMITED
       failure_type:
         title: Failure Type
@@ -3707,6 +3709,9 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/DefaultPaginator"
           - "$ref": "#/definitions/NoPagination"
+      pagination_reset:
+        description: Describes what triggers pagination reset and how to handle it.
+        "$ref": "#/definitions/PaginationReset"
       ignore_stream_slicer_parameters_on_paginated_requests:
         description: If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.
         type: boolean
@@ -3730,6 +3735,36 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  PaginationReset:
+    title: Pagination Reset
+    description: Describes what triggers pagination reset and how to handle it. If SPLIT_USING_CURSOR, the connector developer is accountable for ensuring that the records are returned in ascending order.
+    type: object
+    required:
+      - type
+      - action
+    properties:
+      type:
+        type: string
+        enum: [ PaginationReset ]
+      action:
+        type: string
+        enum:
+          - SPLIT_USING_CURSOR
+          - RESET
+      limits:
+        "$ref": "#/definitions/PaginationResetLimits"
+  PaginationResetLimits:
+    title: Pagination Reset Limits
+    description: Describes the limits that trigger pagination reset
+    type: object
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [ PaginationResetLimits ]
+      number_of_records:
+        type: integer
   GzipDecoder:
     title: gzip
     description: Select 'gzip' for response data that is compressed with gzip. Requires specifying an inner data type/decoder to parse the decompressed data.

{airbyte_cdk-7.3.2 → airbyte_cdk-7.3.3}/airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -151,7 +151,7 @@ class ConcurrentPerPartitionCursor(Cursor):
         self._connector_state_converter = connector_state_converter
         self._cursor_field = cursor_field
 
-        self._cursor_factory = cursor_factory
+        self._cursor_factory = cursor_factory  # self._cursor_factory is flagged as private but is used in model_to_component_factory to ease pagination reset instantiation
         self._partition_router = partition_router
 
         # The dict is ordered to ensure that once the maximum number of partitions is reached,

{airbyte_cdk-7.3.2 → airbyte_cdk-7.3.3}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -539,6 +539,7 @@ class Action(Enum):
     FAIL = "FAIL"
     RETRY = "RETRY"
     IGNORE = "IGNORE"
+    RESET_PAGINATION = "RESET_PAGINATION"
     RATE_LIMITED = "RATE_LIMITED"
 
 
@@ -553,7 +554,14 @@ class HttpResponseFilter(BaseModel):
     action: Optional[Action] = Field(
         None,
         description="Action to execute if a response matches the filter.",
-        examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"],
+        examples=[
+            "SUCCESS",
+            "FAIL",
+            "RETRY",
+            "IGNORE",
+            "RESET_PAGINATION",
+            "RATE_LIMITED",
+        ],
         title="Action",
     )
     failure_type: Optional[FailureType] = Field(
@@ -1173,6 +1181,16 @@ class LegacySessionTokenAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class Action1(Enum):
+    SPLIT_USING_CURSOR = "SPLIT_USING_CURSOR"
+    RESET = "RESET"
+
+
+class PaginationResetLimits(BaseModel):
+    type: Literal["PaginationResetLimits"]
+    number_of_records: Optional[int] = None
+
+
 class CsvDecoder(BaseModel):
     type: Literal["CsvDecoder"]
     encoding: Optional[str] = "utf-8"
@@ -2054,6 +2072,12 @@ class RecordSelector(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class PaginationReset(BaseModel):
+    type: Literal["PaginationReset"]
+    action: Action1
+    limits: Optional[PaginationResetLimits] = None
+
+
 class GzipDecoder(BaseModel):
     type: Literal["GzipDecoder"]
     decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
@@ -2822,6 +2846,10 @@ class SimpleRetriever(BaseModel):
         None,
         description="Paginator component that describes how to navigate through the API's pages.",
     )
+    pagination_reset: Optional[PaginationReset] = Field(
+        None,
+        description="Describes what triggers pagination reset and how to handle it.",
+    )
     ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field(
         False,
         description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.",

{airbyte_cdk-7.3.2 → airbyte_cdk-7.3.3}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -116,11 +116,15 @@ from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_mi
 )
 from airbyte_cdk.sources.declarative.models import (
     CustomStateMigration,
+    PaginationResetLimits,
 )
 from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import (
     DEPRECATION_LOGS_TAG,
     BaseModelWithDeprecations,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    Action1 as PaginationResetActionModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     AddedFieldDefinition as AddedFieldDefinitionModel,
 )
@@ -358,6 +362,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     PageIncrement as PageIncrementModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    PaginationReset as PaginationResetModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ParametrizedComponentsResolver as ParametrizedComponentsResolverModel,
 )
@@ -529,6 +536,7 @@ from airbyte_cdk.sources.declarative.retrievers.file_uploader import (
     LocalFileSystemFileWriter,
     NoopFileWriter,
 )
+from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker
 from airbyte_cdk.sources.declarative.schema import (
     ComplexFieldType,
     DefaultSchemaLoader,
@@ -644,6 +652,8 @@ _NO_STREAM_SLICING = SinglePartitionRouter(parameters={})
 # this would be a circular import
 MAX_SLICES = 5
 
+LOGGER = logging.getLogger(f"airbyte.model_to_component_factory")
+
 
 class ModelToComponentFactory:
     EPOCH_DATETIME_FORMAT = "%s"
@@ -2043,6 +2053,7 @@ class ModelToComponentFactory:
             if isinstance(concurrent_cursor, FinalStateCursor)
             else concurrent_cursor
         )
+
         retriever = self._create_component_from_model(
             model=model.retriever,
             config=config,
@@ -2051,12 +2062,9 @@ class ModelToComponentFactory:
             request_options_provider=request_options_provider,
             stream_slicer=stream_slicer,
             partition_router=partition_router,
-            stop_condition_cursor=concurrent_cursor
-            if self._is_stop_condition_on_cursor(model)
-            else None,
-            client_side_incremental_sync={"cursor": concurrent_cursor}
-            if self._is_client_side_filtering_enabled(model)
-            else None,
+            has_stop_condition_cursor=self._is_stop_condition_on_cursor(model),
+            is_client_side_incremental_sync=self._is_client_side_filtering_enabled(model),
+            cursor=concurrent_cursor,
             transformations=transformations,
             file_uploader=file_uploader,
             incremental_sync=model.incremental_sync,
@@ -3049,7 +3057,7 @@ class ModelToComponentFactory:
         name: str,
         transformations: List[RecordTransformation] | None = None,
         decoder: Decoder | None = None,
-        client_side_incremental_sync: Dict[str, Any] | None = None,
+        client_side_incremental_sync_cursor: Optional[Cursor] = None,
         file_uploader: Optional[DefaultFileUploader] = None,
         **kwargs: Any,
     ) -> RecordSelector:
@@ -3065,14 +3073,14 @@ class ModelToComponentFactory:
         transform_before_filtering = (
             False if model.transform_before_filtering is None else model.transform_before_filtering
         )
-        if client_side_incremental_sync:
+        if client_side_incremental_sync_cursor:
             record_filter = ClientSideIncrementalRecordFilterDecorator(
                 config=config,
                 parameters=model.parameters,
                 condition=model.record_filter.condition
                 if (model.record_filter and hasattr(model.record_filter, "condition"))
                 else None,
-                **client_side_incremental_sync,
+                cursor=client_side_incremental_sync_cursor,
             )
             transform_before_filtering = (
                 True
@@ -3150,8 +3158,9 @@ class ModelToComponentFactory:
         name: str,
         primary_key: Optional[Union[str, List[str], List[List[str]]]],
         request_options_provider: Optional[RequestOptionsProvider] = None,
-        stop_condition_cursor: Optional[Cursor] = None,
-        client_side_incremental_sync: Optional[Dict[str, Any]] = None,
+        cursor: Optional[Cursor] = None,
+        has_stop_condition_cursor: bool = False,
+        is_client_side_incremental_sync: bool = False,
         transformations: List[RecordTransformation],
         file_uploader: Optional[DefaultFileUploader] = None,
         incremental_sync: Optional[
@@ -3181,6 +3190,9 @@ class ModelToComponentFactory:
 
             return _url or _url_base
 
+        if cursor is None:
+            cursor = FinalStateCursor(name, None, self._message_repository)
+
         decoder = (
             self._create_component_from_model(model=model.decoder, config=config)
             if model.decoder
@@ -3192,7 +3204,7 @@ class ModelToComponentFactory:
             config=config,
             decoder=decoder,
             transformations=transformations,
-            client_side_incremental_sync=client_side_incremental_sync,
+            client_side_incremental_sync_cursor=cursor if is_client_side_incremental_sync else None,
             file_uploader=file_uploader,
         )
 
@@ -3280,7 +3292,7 @@ class ModelToComponentFactory:
                 url_base=_get_url(requester),
                 extractor_model=model.record_selector.extractor,
                 decoder=decoder,
-                cursor_used_for_stop_condition=stop_condition_cursor or None,
+                cursor_used_for_stop_condition=cursor if has_stop_condition_cursor else None,
             )
             if model.paginator
             else NoPagination(parameters={})
@@ -3329,6 +3341,13 @@ class ModelToComponentFactory:
                 parameters=model.parameters or {},
             )
 
+        if (
+            model.record_selector.record_filter
+            and model.pagination_reset
+            and model.pagination_reset.limits
+        ):
+            raise ValueError("PaginationResetLimits are not supported while having record filter.")
+
         return SimpleRetriever(
             name=name,
             paginator=paginator,
@@ -3342,9 +3361,40 @@ class ModelToComponentFactory:
             ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests,
             additional_query_properties=query_properties,
             log_formatter=self._get_log_formatter(log_formatter, name),
+            pagination_tracker_factory=self._create_pagination_tracker_factory(
+                model.pagination_reset, cursor
+            ),
             parameters=model.parameters or {},
         )
 
+    def _create_pagination_tracker_factory(
+        self, model: Optional[PaginationResetModel], cursor: Cursor
+    ) -> Callable[[], PaginationTracker]:
+        if model is None:
+            return lambda: PaginationTracker()
+
+        # Until we figure out a way to use any cursor for PaginationTracker, we will have to have this cursor selector logic
+        cursor_factory: Callable[[], Optional[ConcurrentCursor]] = lambda: None
+        if model.action == PaginationResetActionModel.RESET:
+            # in that case, we will let cursor_factory to return None even if the stream has a cursor
+            pass
+        elif model.action == PaginationResetActionModel.SPLIT_USING_CURSOR:
+            if isinstance(cursor, ConcurrentCursor):
+                cursor_factory = lambda: cursor.copy_without_state()  # type: ignore  # the if condition validates that it is a ConcurrentCursor
+            elif isinstance(cursor, ConcurrentPerPartitionCursor):
+                cursor_factory = lambda: cursor._cursor_factory.create(  # type: ignore  # if this becomes a problem, we would need to extract the cursor_factory instantiation logic and make it accessible here
+                    {}, datetime.timedelta(0)
+                )
+            elif not isinstance(cursor, FinalStateCursor):
+                LOGGER.warning(
+                    "Unknown cursor for PaginationTracker. Pagination resets might not work properly"
+                )
+        else:
+            raise ValueError(f"Unknown PaginationReset action: {model.action}")
+
+        limit = model.limits.number_of_records if model and model.limits else None
+        return lambda: PaginationTracker(cursor_factory(), limit)
+
     def _get_log_formatter(
         self, log_formatter: Callable[[Response], Any] | None, name: str
     ) -> Callable[[Response], Any] | None:

{airbyte_cdk-7.3.2 → airbyte_cdk-7.3.3}/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py

@@ -66,14 +66,14 @@ class CompositeErrorHandler(ErrorHandler):
             if not isinstance(matched_error_resolution, ErrorResolution):
                 continue
 
-            if matched_error_resolution.response_action == ResponseAction.SUCCESS:
+            if matched_error_resolution.response_action in [
+                ResponseAction.SUCCESS,
+                ResponseAction.RETRY,
+                ResponseAction.IGNORE,
+                ResponseAction.RESET_PAGINATION,
+            ]:
                 return matched_error_resolution
 
-            if (
-                matched_error_resolution.response_action == ResponseAction.RETRY
-                or matched_error_resolution.response_action == ResponseAction.IGNORE
-            ):
-                return matched_error_resolution
         if matched_error_resolution:
             return matched_error_resolution
 

airbyte_cdk-7.3.3/airbyte_cdk/sources/declarative/retrievers/pagination_tracker.py

@@ -0,0 +1,64 @@
+from typing import Optional
+
+from airbyte_cdk.sources.declarative.models import FailureType
+from airbyte_cdk.sources.declarative.types import Record, StreamSlice
+from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+
+
+class PaginationTracker:
+    _record_count: int
+    _number_of_attempt_with_same_slice: int
+
+    def __init__(
+        self, cursor: Optional[ConcurrentCursor] = None, max_number_of_records: Optional[int] = None
+    ) -> None:
+        """
+        Ideally, we would have passed the `Cursor` interface here instead of `ConcurrentCursor` but not all
+        implementations of `Cursor` can support this use case. For example, if the `ConcurrentPerPartitionCursor`
+        switch to global state, we stop keeping track of the state per partition and therefore can't get an accurate
+        view for a specific stream_slice. In order to solve that, we decided to scope this feature to use only
+        ConcurrentCursor which is the only "leaf" cursor that actually emits stream slices with `cursor_partition`.
+        """
+        self._cursor = cursor
+        self._limit = max_number_of_records
+        self._reset()
+
+        """
+        Given we have a cursor, we do not allow for the same slice to be processed twice because we assume we will
+        always process the same slice.
+
+        Given no cursor, we assume that the pagination reset is for retrying purposes and we allow to retry once.
+        """
+        self._allowed_number_of_attempt_with_same_slice = 1 if self._cursor else 2
+        self._number_of_attempt_with_same_slice = 0
+
+    def observe(self, record: Record) -> None:
+        self._record_count += 1
+        if self._cursor:
+            self._cursor.observe(record)
+
+    def has_reached_limit(self) -> bool:
+        return self._limit is not None and self._record_count >= self._limit
+
+    def _reset(self) -> None:
+        self._record_count = 0
+
+    def reduce_slice_range_if_possible(self, stream_slice: StreamSlice) -> StreamSlice:
+        new_slice = self._cursor.reduce_slice_range(stream_slice) if self._cursor else stream_slice
+
+        if new_slice == stream_slice:
+            self._number_of_attempt_with_same_slice += 1
+            if (
+                self._number_of_attempt_with_same_slice
+                >= self._allowed_number_of_attempt_with_same_slice
+            ):
+                raise AirbyteTracedException(
+                    internal_message=f"There were {self._number_of_attempt_with_same_slice} attempts with the same slice already while the max allowed is {self._allowed_number_of_attempt_with_same_slice}",
+                    failure_type=FailureType.system_error,
+                )
+        else:
+            self._number_of_attempt_with_same_slice = 0
+
+        self._reset()
+        return new_slice

{airbyte_cdk-7.3.2 → airbyte_cdk-7.3.3}/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -3,10 +3,10 @@
 #
 
 import json
+import logging
 from collections import defaultdict
 from dataclasses import InitVar, dataclass, field
 from functools import partial
-from itertools import islice
 from typing import (
     Any,
     Callable,
@@ -39,14 +39,20 @@ from airbyte_cdk.sources.declarative.requesters.request_options import (
     RequestOptionsProvider,
 )
 from airbyte_cdk.sources.declarative.requesters.requester import Requester
+from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker
 from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
 from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer
 from airbyte_cdk.sources.source import ExperimentalClassWarning
+from airbyte_cdk.sources.streams.concurrent.cursor import Cursor
 from airbyte_cdk.sources.streams.core import StreamData
+from airbyte_cdk.sources.streams.http.pagination_reset_exception import (
+    PaginationResetRequiredException,
+)
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
 from airbyte_cdk.utils.mapping_helpers import combine_mappings
 
 FULL_REFRESH_SYNC_COMPLETE_KEY = "__ab_full_refresh_sync_complete"
+LOGGER = logging.getLogger("airbyte")
 
 
 @dataclass
@@ -92,8 +98,14 @@ class SimpleRetriever(Retriever):
     ignore_stream_slicer_parameters_on_paginated_requests: bool = False
     additional_query_properties: Optional[QueryProperties] = None
     log_formatter: Optional[Callable[[requests.Response], Any]] = None
+    pagination_tracker_factory: Callable[[], PaginationTracker] = field(
+        default_factory=lambda: lambda: PaginationTracker()
+    )
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        # while changing `ModelToComponentFactory.create_simple_retriever` to accept a cursor, the sources implementing
+        # a CustomRetriever inheriting for SimpleRetriever needed to have the following validation added.
+        self.cursor = None if isinstance(self.cursor, Cursor) else self.cursor
         self._paginator = self.paginator or NoPagination(parameters=parameters)
         self._parameters = parameters
         self._name = (
@@ -362,90 +374,97 @@ class SimpleRetriever(Retriever):
         stream_state: Mapping[str, Any],
         stream_slice: StreamSlice,
     ) -> Iterable[Record]:
-        pagination_complete = False
-        initial_token = self._paginator.get_initial_token()
-        next_page_token: Optional[Mapping[str, Any]] = (
-            {"next_page_token": initial_token} if initial_token is not None else None
-        )
-        while not pagination_complete:
-            property_chunks: List[List[str]] = (
-                list(
-                    self.additional_query_properties.get_request_property_chunks(
-                        stream_slice=stream_slice
-                    )
-                )
-                if self.additional_query_properties
-                else [
-                    []
-                ]  # A single empty property chunk represents the case where property chunking is not configured
-            )
-
+        pagination_tracker = self.pagination_tracker_factory()
+        reset_pagination = False
+        next_page_token = self._get_initial_next_page_token()
+        while True:
             merged_records: MutableMapping[str, Any] = defaultdict(dict)
             last_page_size = 0
             last_record: Optional[Record] = None
-            response: Optional[requests.Response] = None
-            for properties in property_chunks:
-                if len(properties) > 0:
-                    stream_slice = StreamSlice(
-                        partition=stream_slice.partition or {},
-                        cursor_slice=stream_slice.cursor_slice or {},
-                        extra_fields={"query_properties": properties},
-                    )
-
-                response = self._fetch_next_page(stream_state, stream_slice, next_page_token)
-                for current_record in records_generator_fn(response):
-                    if (
-                        current_record
-                        and self.additional_query_properties
-                        and self.additional_query_properties.property_chunking
+
+            response = None
+            try:
+                if (
+                    self.additional_query_properties
+                    and self.additional_query_properties.property_chunking
+                ):
+                    for properties in self.additional_query_properties.get_request_property_chunks(
+                        stream_slice=stream_slice
                     ):
-                        merge_key = (
-                            self.additional_query_properties.property_chunking.get_merge_key(
-                                current_record
+                        stream_slice = StreamSlice(
+                            partition=stream_slice.partition or {},
+                            cursor_slice=stream_slice.cursor_slice or {},
+                            extra_fields={"query_properties": properties},
+                        )
+                        response = self._fetch_next_page(
+                            stream_state, stream_slice, next_page_token
+                        )
+
+                        for current_record in records_generator_fn(response):
+                            merge_key = (
+                                self.additional_query_properties.property_chunking.get_merge_key(
+                                    current_record
+                                )
                             )
+                            if merge_key:
+                                _deep_merge(merged_records[merge_key], current_record)
+                            else:
+                                # We should still emit records even if the record did not have a merge key
+                                pagination_tracker.observe(current_record)
+                                last_page_size += 1
+                                last_record = current_record
+                                yield current_record
+
+                    for merged_record in merged_records.values():
+                        record = Record(
+                            data=merged_record, stream_name=self.name, associated_slice=stream_slice
                         )
-                        if merge_key:
-                            _deep_merge(merged_records[merge_key], current_record)
-                        else:
-                            # We should still emit records even if the record did not have a merge key
-                            last_page_size += 1
-                            last_record = current_record
-                            yield current_record
-                    else:
+                        pagination_tracker.observe(record)
+                        last_page_size += 1
+                        last_record = record
+                        yield record
+                else:
+                    response = self._fetch_next_page(stream_state, stream_slice, next_page_token)
+                    for current_record in records_generator_fn(response):
+                        pagination_tracker.observe(current_record)
                         last_page_size += 1
                         last_record = current_record
                         yield current_record
-
-            if (
-                self.additional_query_properties
-                and self.additional_query_properties.property_chunking
-            ):
-                for merged_record in merged_records.values():
-                    record = Record(
-                        data=merged_record, stream_name=self.name, associated_slice=stream_slice
-                    )
-                    last_page_size += 1
-                    last_record = record
-                    yield record
-
-            if not response:
-                pagination_complete = True
+            except PaginationResetRequiredException:
+                reset_pagination = True
+            else:
+                if not response:
+                    break
+
+            if reset_pagination or pagination_tracker.has_reached_limit():
+                next_page_token = self._get_initial_next_page_token()
+                previous_slice = stream_slice
+                stream_slice = pagination_tracker.reduce_slice_range_if_possible(stream_slice)
+                LOGGER.info(
+                    f"Hitting PaginationReset event. StreamSlice used will go from {previous_slice} to {stream_slice}"
+                )
+                reset_pagination = False
             else:
                 last_page_token_value = (
                     next_page_token.get("next_page_token") if next_page_token else None
                 )
                 next_page_token = self._next_page_token(
-                    response=response,
+                    response=response,  # type:ignore # we are breaking from the loop on the try/else if there are no response so this should be fine
                     last_page_size=last_page_size,
                     last_record=last_record,
                     last_page_token_value=last_page_token_value,
                 )
                 if not next_page_token:
-                    pagination_complete = True
+                    break
 
         # Always return an empty generator just in case no records were ever yielded
         yield from []
 
+    def _get_initial_next_page_token(self) -> Optional[Mapping[str, Any]]:
+        initial_token = self._paginator.get_initial_token()
+        next_page_token = {"next_page_token": initial_token} if initial_token is not None else None
+        return next_page_token
+
     def _read_single_page(
         self,
         records_generator_fn: Callable[[Optional[requests.Response]], Iterable[Record]],