airbyte-cdk 6.43.1__py3-none-any.whl → 6.45.0__py3-none-any.whl

airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -1,8 +1,9 @@
 #
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
 
 import json
+from collections import defaultdict
 from dataclasses import InitVar, dataclass, field
 from functools import partial
 from itertools import islice
@@ -12,6 +13,7 @@ from typing import (
     Iterable,
     List,
     Mapping,
+    MutableMapping,
     Optional,
     Set,
     Tuple,
@@ -31,6 +33,7 @@ from airbyte_cdk.sources.declarative.partition_routers.single_partition_router i
 )
 from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination
 from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
+from airbyte_cdk.sources.declarative.requesters.query_properties import QueryProperties
 from airbyte_cdk.sources.declarative.requesters.request_options import (
     DefaultRequestOptionsProvider,
     RequestOptionsProvider,
@@ -88,6 +91,7 @@ class SimpleRetriever(Retriever):
     )
     cursor: Optional[DeclarativeCursor] = None
     ignore_stream_slicer_parameters_on_paginated_requests: bool = False
+    additional_query_properties: Optional[QueryProperties] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._paginator = self.paginator or NoPagination(parameters=parameters)
@@ -445,43 +449,110 @@ class SimpleRetriever(Retriever):
         :param stream_slice: The stream slice to read data for
         :return: The records read from the API source
         """
-        _slice = stream_slice or StreamSlice(partition={}, cursor_slice={})  # None-check
 
-        most_recent_record_from_slice = None
-        record_generator = partial(
-            self._parse_records,
-            stream_slice=stream_slice,
-            stream_state=self.state or {},
-            records_schema=records_schema,
+        property_chunks = (
+            list(
+                self.additional_query_properties.get_request_property_chunks(
+                    stream_slice=stream_slice
+                )
+            )
+            if self.additional_query_properties
+            else []
         )
+        records_without_merge_key = []
+        merged_records: MutableMapping[str, Any] = defaultdict(dict)
 
-        if self.cursor and isinstance(self.cursor, ResumableFullRefreshCursor):
-            stream_state = self.state
-
-            # Before syncing the RFR stream, we check if the job's prior attempt was successful and don't need to
-            # fetch more records. The platform deletes stream state for full refresh streams before starting a
-            # new job, so we don't need to worry about this value existing for the initial attempt
-            if stream_state.get(FULL_REFRESH_SYNC_COMPLETE_KEY):
-                return
+        _slice = stream_slice or StreamSlice(partition={}, cursor_slice={})  # None-check
+        most_recent_record_from_slice = None
 
-            yield from self._read_single_page(record_generator, stream_state, _slice)
-        else:
-            for stream_data in self._read_pages(record_generator, self.state, _slice):
-                current_record = self._extract_record(stream_data, _slice)
-                if self.cursor and current_record:
-                    self.cursor.observe(_slice, current_record)
-
-                # Latest record read, not necessarily within slice boundaries.
-                # TODO Remove once all custom components implement `observe` method.
-                # https://github.com/airbytehq/airbyte-internal-issues/issues/6955
-                most_recent_record_from_slice = self._get_most_recent_record(
-                    most_recent_record_from_slice, current_record, _slice
+        if self.additional_query_properties:
+            for properties in property_chunks:
+                _slice = StreamSlice(
+                    partition=_slice.partition or {},
+                    cursor_slice=_slice.cursor_slice or {},
+                    extra_fields={"query_properties": properties},
+                )  # None-check
+
+                record_generator = partial(
+                    self._parse_records,
+                    stream_slice=_slice,
+                    stream_state=self.state or {},
+                    records_schema=records_schema,
                 )
-                yield stream_data
 
+                for stream_data in self._read_pages(record_generator, self.state, _slice):
+                    current_record = self._extract_record(stream_data, _slice)
+                    if self.cursor and current_record:
+                        self.cursor.observe(_slice, current_record)
+
+                    # Latest record read, not necessarily within slice boundaries.
+                    # TODO Remove once all custom components implement `observe` method.
+                    # https://github.com/airbytehq/airbyte-internal-issues/issues/6955
+                    most_recent_record_from_slice = self._get_most_recent_record(
+                        most_recent_record_from_slice, current_record, _slice
+                    )
+
+                    if current_record and self.additional_query_properties.property_chunking:
+                        merge_key = (
+                            self.additional_query_properties.property_chunking.get_merge_key(
+                                current_record
+                            )
+                        )
+                        if merge_key:
+                            merged_records[merge_key].update(current_record)
+                        else:
+                            # We should still emit records even if the record did not have a merge key
+                            records_without_merge_key.append(current_record)
+                    else:
+                        yield stream_data
             if self.cursor:
                 self.cursor.close_slice(_slice, most_recent_record_from_slice)
-        return
+
+            if len(merged_records) > 0:
+                yield from [
+                    Record(data=merged_record, stream_name=self.name, associated_slice=stream_slice)
+                    for merged_record in merged_records.values()
+                ]
+            if len(records_without_merge_key) > 0:
+                yield from records_without_merge_key
+        else:
+            _slice = stream_slice or StreamSlice(partition={}, cursor_slice={})  # None-check
+
+            most_recent_record_from_slice = None
+            record_generator = partial(
+                self._parse_records,
+                stream_slice=stream_slice,
+                stream_state=self.state or {},
+                records_schema=records_schema,
+            )
+
+            if self.cursor and isinstance(self.cursor, ResumableFullRefreshCursor):
+                stream_state = self.state
+
+                # Before syncing the RFR stream, we check if the job's prior attempt was successful and don't need to
+                # fetch more records. The platform deletes stream state for full refresh streams before starting a
+                # new job, so we don't need to worry about this value existing for the initial attempt
+                if stream_state.get(FULL_REFRESH_SYNC_COMPLETE_KEY):
+                    return
+
+                yield from self._read_single_page(record_generator, stream_state, _slice)
+            else:
+                for stream_data in self._read_pages(record_generator, self.state, _slice):
+                    current_record = self._extract_record(stream_data, _slice)
+                    if self.cursor and current_record:
+                        self.cursor.observe(_slice, current_record)
+
+                    # Latest record read, not necessarily within slice boundaries.
+                    # TODO Remove once all custom components implement `observe` method.
+                    # https://github.com/airbytehq/airbyte-internal-issues/issues/6955
+                    most_recent_record_from_slice = self._get_most_recent_record(
+                        most_recent_record_from_slice, current_record, _slice
+                    )
+                    yield stream_data
+
+                if self.cursor:
+                    self.cursor.close_slice(_slice, most_recent_record_from_slice)
+            return
 
     def _get_most_recent_record(
         self,

{airbyte_cdk-6.43.1.dist-info → airbyte_cdk-6.45.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.43.1
+Version: 6.45.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.43.1.dist-info → airbyte_cdk-6.45.0.dist-info}/RECORD

@@ -61,9 +61,9 @@ airbyte_cdk/sources/declarative/auth/oauth.py,sha256=SUfib1oSzlyRRnOSg8Bui73mfyr
 airbyte_cdk/sources/declarative/auth/selective_authenticator.py,sha256=qGwC6YsCldr1bIeKG6Qo-A9a5cTdHw-vcOn3OtQrS4c,1540
 airbyte_cdk/sources/declarative/auth/token.py,sha256=2EnE78EhBOY9hbeZnQJ9AuFaM-G7dccU-oKo_LThRQk,11070
 airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=Jzuxlmt1_-_aFC_n0OmP8L1nDOacLzbEVVx3kjdX_W8,3104
-airbyte_cdk/sources/declarative/checks/__init__.py,sha256=nsVV5Bo0E_tBNd8A4Xdsdb-75PpcLo5RQu2RQ_Gv-ME,806
+airbyte_cdk/sources/declarative/checks/__init__.py,sha256=cpoBwEbRNcMi7Rmi5pD63ppUvAOZsAWzasmc57ob9rc,873
 airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py,sha256=HUktywjI8pqOeED08UGqponUSwxs2TOAECTowlWlrRE,2138
-airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQrilWCfJmncBzXCZ18ptRNip3XA,2139
+airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=QeExVmpSYjr_CnghHuJBn5oHW6wyKmAr7rotSSfcWp8,7145
 airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
 airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
 airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
@@ -71,7 +71,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=uhy0dRkA
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=twYvmNco0B7R8N5QXcTc1tbqWFB_7b-_i149LJbMss8,153129
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=pGFwleIC0sXFY-a6QphWyjY9QjUfHasgChPAMZFWjU8,158152
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
@@ -114,13 +114,13 @@ airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=nzQqJjBo8wFUF9gqF9E52zDsEDQGpG46aWylViTcAJU,108372
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=EfhCFVz_UHl7_cbRlf5hvFO1J5osxAZjISeqgbXE2t8,112007
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=4C15MKV-zOrMVQAm4FyohDsrJUBCSpMv5tZw0SK3aeI,9685
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=APJkP5dmDU4aIaj7w3quGjrP1cV3MMp2gxbTckhOVRA,149720
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Qim2qR64SKWbxD-Au0KAB3fLf6G2YfppUv3Q0w4QYtE,158619
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -156,13 +156,20 @@ airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_incremen
 airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py,sha256=Z2i6a-oKMmOTxHxsTVSnyaShkJ3u8xZw1xIJdx2yxss,2731
 airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py,sha256=ZBshGQNr5Bb_V8dqnWRISqdXFcjm1CKIXnlfbRhNl8g,1308
 airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py,sha256=LoKXdUbSgHEtSwtA8DFrnX6SpQbRVVwreY8NguTKTcI,2229
+airbyte_cdk/sources/declarative/requesters/query_properties/__init__.py,sha256=sHwHVuN6djuRBF7zQb-HmINV0By4wE5j_i6TjmIPMzQ,494
+airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py,sha256=3h9Ae6TNGagh9sMYWdG5KoEFWDlqUWZ5fkswTPreveM,1616
+airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py,sha256=YmUeeY3ZpsuK2VTF3SkdVuJcplI1I4UfhgzOrggifag,2748
+airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py,sha256=2VWhgphAFKmHJhzp-UoSP9_QR3eYOLPT0nzMDyglBV4,2650
+airbyte_cdk/sources/declarative/requesters/query_properties/strategies/__init__.py,sha256=ojiPj9eVU7SuNpF3RZwhZWW21IYLQYEoxpzg1rCdvNM,350
+airbyte_cdk/sources/declarative/requesters/query_properties/strategies/group_by_key.py,sha256=np4uTwSpQvXxubIzVbwSDX0Xf3EgVn8kkhs6zYLOdAQ,1081
+airbyte_cdk/sources/declarative/requesters/query_properties/strategies/merge_strategy.py,sha256=iuk9QxpwvKVtdrq9eadQVkZ-Sfk3qhyyAAErprBfw2s,516
 airbyte_cdk/sources/declarative/requesters/request_option.py,sha256=Bl0gxGWudmwT3FXBozTN00WYle2jd6ry_S1YylCnwqM,4825
 airbyte_cdk/sources/declarative/requesters/request_options/__init__.py,sha256=WCwpKqM4wKqy-DHJaCHbKAlFqRVOqMi9K5qonxIfi_Y,809
 airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py,sha256=31nG6_0igidJFQon37-WeQkTpG3g2A5ZmlluI3ilZdE,3632
 airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py,sha256=SRROdPJZ5kuqHLOlkh115pWP9nDGfDxRYPgH9oD3hPo,1798
 airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py,sha256=86YozYuBDfu0t9NbevIvQoGU0vqTP4rt3dRSTsHz3PA,2269
 airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py,sha256=rR00kE64U2yL0McU1gPr4_W5_sLUqwDgL3Nvj691nRU,2884
-airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py,sha256=vOsdHfWHiTFc89WENHPv1hcxLgdzycMXVT_IEtLuhfs,5012
+airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py,sha256=dRlG1IyEOVzWFw7wm-8TBPn7JUtZw3jz6oAoH5yuuf0,6375
 airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py,sha256=8YRiDzjYvqJ-aMmKFcjqzv_-e8OZ5QG_TbpZ-nuCu6s,2590
 airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
 airbyte_cdk/sources/declarative/requesters/requester.py,sha256=OcDzuCBgD1owK_lBPG0KbRRHRn9kzbuRveU4HejDiv4,5116
@@ -173,7 +180,7 @@ airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=Aio
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=nQepwG_RfW53sgwvK5dLPqfCx0VjsQ83nYoPjBMAaLM,527
 airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=6oZtnCHm9NdDvjTSrVwPQOXGSdETSIR7eWH2vFjM7jI,4855
 airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
-airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=p6O4FYS7zzPq6uQT2NVnughUjI66tePaXVlyhCAyyv0,27746
+airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=Ixs_byM-DMfYiYB-iOa97yH4WGAvxgTdTYeAsskHdYM,31134
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
 airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=KTACrIE23a83wsm3Rd9Eb4K6-20lrGqYxTHNp9yxsso,1820
 airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=J8Q_iJYhcSQLWyt0bTZCbDAGpxt9G8FCc6Q9jtGsNzw,10703
@@ -359,9 +366,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.43.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.43.1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.43.1.dist-info/METADATA,sha256=Zrfti0WPq-_VsjOASQALX5Gctg83yqECVMd-zOsyLOM,6071
-airbyte_cdk-6.43.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.43.1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.43.1.dist-info/RECORD,,
+airbyte_cdk-6.45.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.45.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.45.0.dist-info/METADATA,sha256=QFEPcdQP0d_cCYIUW4uSBOL-muBtxr0wu7kbmc65LkQ,6071
+airbyte_cdk-6.45.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.45.0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.45.0.dist-info/RECORD,,