airbyte-cdk 0.64.1__py3-none-any.whl → 0.65.0__py3-none-any.whl

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -2001,6 +2001,10 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/DefaultPaginator"
           - "$ref": "#/definitions/NoPagination"
+      ignore_stream_slicer_parameters_on_paginated_requests:
+        description: If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.
+        type: boolean
+        default: false
       partition_router:
         title: Partition Router
         description: PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1338,6 +1338,10 @@ class SimpleRetriever(BaseModel):
         None,
         description="Paginator component that describes how to navigate through the API's pages.",
     )
+    ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field(
+        False,
+        description='If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.',
+    )
     partition_router: Optional[
         Union[
             CustomPartitionRouter,

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -958,12 +958,17 @@ class ModelToComponentFactory:
         cursor_used_for_stop_condition = cursor if stop_condition_on_cursor else None
         paginator = (
             self._create_component_from_model(
-                model=model.paginator, config=config, url_base=url_base, cursor_used_for_stop_condition=cursor_used_for_stop_condition
+                model=model.paginator,
+                config=config,
+                url_base=url_base,
+                cursor_used_for_stop_condition=cursor_used_for_stop_condition,
             )
             if model.paginator
             else NoPagination(parameters={})
         )
 
+        ignore_stream_slicer_parameters_on_paginated_requests = model.ignore_stream_slicer_parameters_on_paginated_requests or False
+
         if self._limit_slices_fetched or self._emit_connector_builder_messages:
             return SimpleRetrieverTestReadDecorator(
                 name=name,
@@ -975,6 +980,7 @@ class ModelToComponentFactory:
                 cursor=cursor,
                 config=config,
                 maximum_number_of_slices=self._limit_slices_fetched or 5,
+                ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests,
                 parameters=model.parameters or {},
             )
         return SimpleRetriever(
@@ -986,6 +992,7 @@ class ModelToComponentFactory:
             stream_slicer=stream_slicer,
             cursor=cursor,
             config=config,
+            ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests,
             parameters=model.parameters or {},
         )
 

airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -59,6 +59,7 @@ class SimpleRetriever(Retriever):
     paginator: Optional[Paginator] = None
     stream_slicer: StreamSlicer = SinglePartitionRouter(parameters={})
     cursor: Optional[Cursor] = None
+    ignore_stream_slicer_parameters_on_paginated_requests: bool = False
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._paginator = self.paginator or NoPagination(parameters=parameters)
@@ -105,12 +106,12 @@ class SimpleRetriever(Retriever):
         Returned merged mapping otherwise
         """
         # FIXME we should eventually remove the usage of stream_state as part of the interpolation
-        return combine_mappings(
-            [
-                paginator_method(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
-                stream_slicer_method(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
-            ]
-        )
+        mappings = [
+            paginator_method(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token),
+        ]
+        if not next_page_token or not self.ignore_stream_slicer_parameters_on_paginated_requests:
+            mappings.append(stream_slicer_method(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token))
+        return combine_mappings(mappings)
 
     def _request_headers(
         self,

{airbyte_cdk-0.64.1.dist-info → airbyte_cdk-0.65.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.64.1
+Version: 0.65.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.64.1.dist-info → airbyte_cdk-0.65.0.dist-info}/RECORD

@@ -38,7 +38,7 @@ airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py
 airbyte_cdk/sources/concurrent_source/thread_pool_manager.py,sha256=hFj5rsRtORurl3fwH8GC9h6Uz2wbzBFOLWUxJ-YJ7J8,4801
 airbyte_cdk/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/create_partial.py,sha256=sUJOwD8hBzW4pxw2XhYlSTMgl-WMc5WpP5Oq_jo3fHw,3371
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=Txos1VS1yQrH1c1Wx3zXPhfsNz0fugfuiKN73cJGswI,87608
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=KkWDEq_Aw1Zu7BwPBp-2K6jM74kVmgSe5rHsk5lj5_A,87906
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=U2As9PDKmcWDgbsWUo-RetJ9fxQOBlwntWZ0NOgs5Ac,1453
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=3WADuP_urkJVoMZiBHN1Dw1x-KqOO1YlU030Oo9aLWs,5868
 airbyte_cdk/sources/declarative/exceptions.py,sha256=kTPUA4I2NV4J6HDz-mKPGMrfuc592akJnOyYx38l_QM,176
@@ -80,14 +80,14 @@ airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=dyIM-bzh54
 airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=8bUH6xJRkao8BanwzBFj-1CDj7RR2xPO5u_-FfyRNks,5128
 airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=V6WGKJ9cXX1rjuM4bK3Cs9xEryMlkY2U3FMsSBhrgC8,3098
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=EiYnzwCHZV7EYqMJqcy6xKSeHvTKZBsQndjbEwmiTW4,93
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=BEWjICF1adVdUHmRJrzTO05oQNr98N-z06PeNsVnS8U,60142
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=jqn0yYJSSVeokaG7xxgCyr1wpJrC4CsA3D0Pf7g9LI8,60440
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/class_types_registry.py,sha256=5vOvMuyWlpALrOq2ehLxa7wO6tlFIlgUNtMYrMCKIjE,6092
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=y7_G5mM07zxT5YG975kdC2PAja-Uc83pYp8WrV3GNdo,522
 airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py,sha256=W8BcK4KOg4ifNXgsdeIoV4oneHjXBKcPHEZHIC4r-hM,3801
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=i2yUdrdlPUHI0dSQX0zBT8WSg912SMiCF8qkQ8VvlA4,8287
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=6ukHx0bBrCJm9rek1l_MEfS3U_gdJcM4pJRyifJEOp0,6412
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=yykS21Bt27iYKL_F-doFOuKDMl7T5D5zoPsIlxBuaik,58456
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=yzJZx5bJVxUPWojKm3khM-ABRiVHlmXyBZGnRg0aV-A,58885
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=27sOWhw2LBQs62HchURakHQ2M_mtnOatNgU6q8RUtpU,476
 airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=UMK5B3vVN9DYGyMfDkMmeHGF2jgm0iQ_NSggO5v_zqw,4122
 airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py,sha256=CAbyL0CJRReHZtsyyvZd7X0zE2ELt5CLnE4feDJkx2Y,1575
@@ -128,7 +128,7 @@ airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_
 airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py,sha256=zuHp-yfvjAF6w5zARo2LCkds-HqSRT6bXfNkDbAmq7s,2645
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=IiHXDeKtibRqeWcRUckmSiXfk--u-sFMw3APWK8PCGQ,339
 airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=pwW9BSbWOiCg8FtREYPI_cE5KfAFV3Z9sXqER1HXP_o,1677
-airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=WrKy8j3ASQBi5bF2nzYR_6wVoivBRkh2T65Cxdk-x2s,19050
+airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=8JGxPsIqTyScVHOVDixcc1tDESqf9JXvD8yPxO_0Td8,19230
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=ul8L9S0-__AMEdbCLHBq-PMEeA928NVp8BB83BMotfU,517
 airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=t0ll098cIG2Wr1rq1rZ3QDZ9WnScUuqAh42YVoTRWrU,1794
 airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRsatRJq3R3BeyRR0wIoK3gcP1gcpVRQ_P5U,464
@@ -358,7 +358,7 @@ unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py,sha2
 unit_tests/sources/declarative/requesters/request_options/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py,sha256=bjcaTb8I37tBhs5b_FLRTLkDZAmKjGRywpcN4oGl-zI,5900
 unit_tests/sources/declarative/retrievers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
-unit_tests/sources/declarative/retrievers/test_simple_retriever.py,sha256=PUGVi5eCb9A6amgTinyi15fVbXh33Inlo5o72jFfSvU,18117
+unit_tests/sources/declarative/retrievers/test_simple_retriever.py,sha256=mDF2m-2hqTr7VkrdoIffE8vMaUU9sv5D4At0DVtgEJw,20273
 unit_tests/sources/declarative/schema/__init__.py,sha256=i-iWyCqXPVgY-4miy16FH8U06gW_1_49AVq_8S8rVWY,134
 unit_tests/sources/declarative/schema/test_default_schema_loader.py,sha256=cWOFJnT9fhcEU6XLHkoe3E83mCjWc8lEttT0PFcvAm8,1091
 unit_tests/sources/declarative/schema/test_inline_schema_loader.py,sha256=vDJauhZ8og8M9ZqKDbf12SSYSfhUZ0_LmH7zjJHCHwI,517
@@ -456,8 +456,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
 unit_tests/utils/test_secret_utils.py,sha256=CdKK8A2-5XVxbXVtX22FK9dwwMeP5KNqDH6luWRXSNw,5256
 unit_tests/utils/test_stream_status_utils.py,sha256=Xr8MZ2HWgTVIyMbywDvuYkRaUF4RZLQOT8-JjvcfR24,2970
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.64.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.64.1.dist-info/METADATA,sha256=4MivqzRAHUdF_DPyDk0hkux67sY5xKT1CWapksto6y4,11072
-airbyte_cdk-0.64.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-airbyte_cdk-0.64.1.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.64.1.dist-info/RECORD,,
+airbyte_cdk-0.65.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.65.0.dist-info/METADATA,sha256=OSX5OQNNDpUHP7s_NHPfD7FG58xiBvJEYhgCd10mKd4,11072
+airbyte_cdk-0.65.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+airbyte_cdk-0.65.0.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.65.0.dist-info/RECORD,,

unit_tests/sources/declarative/retrievers/test_simple_retriever.py

@@ -273,6 +273,46 @@ def test_get_request_headers(test_name, paginator_mapping, expected_mapping):
                 pass
 
 
+@pytest.mark.parametrize(
+    "test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping",
+    [
+        ("test_do_not_ignore_stream_slicer_params_if_ignore_is_true_but_no_next_page_token", {"key_from_pagination": "1000"}, True, None, {"key_from_pagination": "1000"}),
+        ("test_do_not_ignore_stream_slicer_params_if_ignore_is_false_and_no_next_page_token", {"key_from_pagination": "1000"}, False, None, {"key_from_pagination": "1000", "key_from_slicer": "value"}),
+        ("test_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, True, {"page": 2}, {"key_from_pagination": "1000"}),
+        ("test_do_not_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, False, {"page": 2}, {"key_from_pagination": "1000", "key_from_slicer": "value"}),
+    ],
+)
+def test_ignore_stream_slicer_parameters_on_paginated_requests(test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping):
+    # This test is separate from the other request options because request headers must be strings
+    paginator = MagicMock()
+    paginator.get_request_headers.return_value = paginator_mapping
+    requester = MagicMock(use_cache=False)
+
+    stream_slicer = MagicMock()
+    stream_slicer.get_request_headers.return_value = {"key_from_slicer": "value"}
+
+    record_selector = MagicMock()
+    retriever = SimpleRetriever(
+        name="stream_name",
+        primary_key=primary_key,
+        requester=requester,
+        record_selector=record_selector,
+        stream_slicer=stream_slicer,
+        paginator=paginator,
+        ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests,
+        parameters={},
+        config={},
+    )
+
+    request_option_type_to_method = {
+        RequestOptionType.header: retriever._request_headers,
+    }
+
+    for _, method in request_option_type_to_method.items():
+        actual_mapping = method(None, None, next_page_token={"next_page_token": "1000"})
+        assert expected_mapping == actual_mapping
+
+
 @pytest.mark.parametrize(
     "test_name, slicer_body_data, paginator_body_data, expected_body_data",
     [