airbyte-cdk 0.52.7__py3-none-any.whl → 0.52.8__py3-none-any.whl

airbyte_cdk/sources/streams/core.py

@@ -12,9 +12,10 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
 
 import airbyte_cdk.sources.utils.casing as casing
 from airbyte_cdk.models import AirbyteMessage, AirbyteStream, SyncMode
+from airbyte_cdk.models import Type as MessageType
 
 # list of all possible HTTP methods which can be used for sending of request bodies
-from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
+from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader
 from airbyte_cdk.sources.utils.slice_logger import SliceLogger
 from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
 from deprecated.classic import deprecated
@@ -123,6 +124,57 @@ class Stream(ABC):
                 cursor_field=cursor_field,
             )
 
+    def read_incremental(  # type: ignore  # ignoring typing for ConnectorStateManager because of circular dependencies
+        self,
+        cursor_field: Optional[List[str]],
+        logger: logging.Logger,
+        slice_logger: SliceLogger,
+        stream_state: MutableMapping[str, Any],
+        state_manager,
+        per_stream_state_enabled: bool,
+        internal_config: InternalConfig,
+    ) -> Iterable[StreamData]:
+        slices = self.stream_slices(
+            cursor_field=cursor_field,
+            sync_mode=SyncMode.incremental,
+            stream_state=stream_state,
+        )
+        logger.debug(f"Processing stream slices for {self.name} (sync_mode: incremental)", extra={"stream_slices": slices})
+
+        has_slices = False
+        record_counter = 0
+        for _slice in slices:
+            has_slices = True
+            if slice_logger.should_log_slice_message(logger):
+                yield slice_logger.create_slice_log_message(_slice)
+            records = self.read_records(
+                sync_mode=SyncMode.incremental,
+                stream_slice=_slice,
+                stream_state=stream_state,
+                cursor_field=cursor_field or None,
+            )
+            for record_data_or_message in records:
+                yield record_data_or_message
+                if isinstance(record_data_or_message, Mapping) or (
+                    hasattr(record_data_or_message, "type") and record_data_or_message.type == MessageType.RECORD
+                ):
+                    record_data = record_data_or_message if isinstance(record_data_or_message, Mapping) else record_data_or_message.record
+                    stream_state = self.get_updated_state(stream_state, record_data)
+                    checkpoint_interval = self.state_checkpoint_interval
+                    record_counter += 1
+                    if checkpoint_interval and record_counter % checkpoint_interval == 0:
+                        yield self._checkpoint_state(stream_state, state_manager, per_stream_state_enabled)
+
+                    if internal_config.is_limit_reached(record_counter):
+                        break
+
+            yield self._checkpoint_state(stream_state, state_manager, per_stream_state_enabled)
+
+        if not has_slices:
+            # Safety net to ensure we always emit at least one state message even if there are no slices
+            checkpoint = self._checkpoint_state(stream_state, state_manager, per_stream_state_enabled)
+            yield checkpoint
+
     @abstractmethod
     def read_records(
         self,
@@ -304,3 +356,21 @@ class Stream(ABC):
             return wrapped_keys
         else:
             raise ValueError(f"Element must be either list or str. Got: {type(keys)}")
+
+    def _checkpoint_state(  # type: ignore  # ignoring typing for ConnectorStateManager because of circular dependencies
+        self,
+        stream_state: Mapping[str, Any],
+        state_manager,
+        per_stream_state_enabled: bool,
+    ) -> AirbyteMessage:
+        # First attempt to retrieve the current state using the stream's state property. We receive an AttributeError if the state
+        # property is not implemented by the stream instance and as a fallback, use the stream_state retrieved from the stream
+        # instance's deprecated get_updated_state() method.
+        try:
+            state_manager.update_state_for_stream(
+                self.name, self.namespace, self.state  # type: ignore # we know the field might not exist...
+            )
+
+        except AttributeError:
+            state_manager.update_state_for_stream(self.name, self.namespace, stream_state)
+        return state_manager.create_state_message(self.name, self.namespace, send_per_stream_state=per_stream_state_enabled)

{airbyte_cdk-0.52.7.dist-info → airbyte_cdk-0.52.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.52.7
+Version: 0.52.8
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.52.7.dist-info → airbyte_cdk-0.52.8.dist-info}/RECORD

@@ -13,7 +13,7 @@ airbyte_cdk/connector_builder/models.py,sha256=U2LrL1syxZ0gQ3LgnwVj9ozL6uGH5f9bi
 airbyte_cdk/destinations/__init__.py,sha256=0Uxmz3iBAyZJdk_bqUVt2pb0UwRTpFjTnFE6fQFbWKY,126
 airbyte_cdk/destinations/destination.py,sha256=_tIMnKcRQbtIsjVvNOVjfbIxgCNLuBXQwQj8MyVm3BI,5420
 airbyte_cdk/destinations/vector_db_based/__init__.py,sha256=eAkzwTjBbXBhJ5GfPO5I53Zgpv5xQFLRQS8n4nuyPt0,1006
-airbyte_cdk/destinations/vector_db_based/config.py,sha256=HXzXnItahHfpyzQzwcNrypW8mmzRsJTNd4v1GB-z6TU,9543
+airbyte_cdk/destinations/vector_db_based/config.py,sha256=tMp8blgdrI4t7a9Ri9Vydk0TOcRqLTHHUjVlXtc0Wa4,9562
 airbyte_cdk/destinations/vector_db_based/document_processor.py,sha256=ldrlmCT4gFHc_A5B_um4OteXg1OR0LGyDmswO1316tA,8649
 airbyte_cdk/destinations/vector_db_based/embedder.py,sha256=davAE4UtrpWDjbV74tck5zvKksxizvSdF9X51WFMbW4,10913
 airbyte_cdk/destinations/vector_db_based/indexer.py,sha256=58Uf34yIe0QHbnpbkS7rH2sqL7eLzwWUjx7X4yciyeA,3165
@@ -24,11 +24,11 @@ airbyte_cdk/models/__init__.py,sha256=Kg8YHBqUsNWHlAw-u3ZGdG4dxLh7qBlHhqMRfamNCR
 airbyte_cdk/models/airbyte_protocol.py,sha256=DoJvnmGM3xMAZFTwA6_RGMiKSFqfE3ib_Ru0KJ65Ag4,100
 airbyte_cdk/models/well_known_types.py,sha256=KKfNbow2gdLoC1Z4hcXy_JR8m_acsB2ol7gQuEgjobw,117
 airbyte_cdk/sources/__init__.py,sha256=Ov7Uf03KPSZUmMZqZfUAK3tQwsdKjDQUDvTb-H0JyfA,1141
-airbyte_cdk/sources/abstract_source.py,sha256=0YvschwQsz3EgVaMw-WlV3WcM12_VF72MKRXaCSxaz4,16428
+airbyte_cdk/sources/abstract_source.py,sha256=iw4Y5tZtyXsY0XDVPNiQiAZBp6O_zyqRMsOEhu7I7lo,13375
 airbyte_cdk/sources/config.py,sha256=PYsY7y2u3EUwxLiEb96JnuKwH_E8CuxKggsRO2ZPSRc,856
 airbyte_cdk/sources/connector_state_manager.py,sha256=wsmUgII398MazCTKxwLBLzeiU6Z-tMTrKX882EEy-YE,10904
 airbyte_cdk/sources/http_logger.py,sha256=v0kkpDtA0GUOgj6_3AayrYaBrSHBqG4t3MGbrtxaNmU,1437
-airbyte_cdk/sources/source.py,sha256=N3vHZzdUsBETFsql-YpO-LcgjolT_jcnAuHBhGD6Hqk,4278
+airbyte_cdk/sources/source.py,sha256=dk50z8Roc28MJ8FxWe652B-GwItO__bTZqFm7WOtHnw,4412
 airbyte_cdk/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/create_partial.py,sha256=sUJOwD8hBzW4pxw2XhYlSTMgl-WMc5WpP5Oq_jo3fHw,3371
 airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=LtLvEpzKo86RzMO6n20-z4ECW6P0Yoi26HXRCSLP9K0,85049
@@ -181,24 +181,25 @@ airbyte_cdk/sources/file_based/stream/cursor/__init__.py,sha256=MhFB5hOo8sjwvCh8
 airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py,sha256=i-FPeK8lwCzX34GCcmvL5Yvdh8-uu7FeCVYDoFbD7IY,1920
 airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py,sha256=kuJRKgDYOGXRk0V0I8BpFxg0hGv7SfV_nBpmmn45F88,6815
 airbyte_cdk/sources/message/__init__.py,sha256=14ZSLah9uyI_CyK7_jIyq521vlgKAdihe6Ciw6-jLgU,372
-airbyte_cdk/sources/message/repository.py,sha256=kflbIkUwCWXMKpe6566TD_HRjRqEZKQ0h2RpxzjWLJk,4994
+airbyte_cdk/sources/message/repository.py,sha256=tQOmtWxrAp1CMiOKi5SdIEWzcmgnCUYd-xL3fcupUT4,4583
 airbyte_cdk/sources/singer/__init__.py,sha256=D3zQSiWT0B9t0kKE4JPZjrcDnP2YnFNJ3dfYqSaxo9w,246
 airbyte_cdk/sources/singer/singer_helpers.py,sha256=q1LmgjFxSnN-dobMy7nikUwcK-9FvW5QQfgTqiclbAE,15649
 airbyte_cdk/sources/singer/source.py,sha256=3YY8UTOXmctvMVUnYmIegmL3_IxF55iGP_bc_s2MZdY,8530
 airbyte_cdk/sources/streams/__init__.py,sha256=IztrWN5IU_N5GCKDyRSEuoWdZohFTcgIbAIkaCFkr_Q,176
 airbyte_cdk/sources/streams/availability_strategy.py,sha256=7BM0qLvXS0QrlKvnVkBEw4Cw8i7PCENCBLcIAcuD3nY,1007
-airbyte_cdk/sources/streams/core.py,sha256=3tnL0JBtzasuGER_kTY7CR7soSgChDvjCuPuw241HOM,12442
+airbyte_cdk/sources/streams/core.py,sha256=bIuQV7Zs9JpIyNDcfPCbyzv-BWDr_2ictK7s5AihLZQ,16025
 airbyte_cdk/sources/streams/concurrent/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/streams/concurrent/abstract_stream.py,sha256=GCcRvUixoDOkNPy0vK37xdGxYaOfZXxVH6InzndduaE,3525
-airbyte_cdk/sources/streams/concurrent/adapters.py,sha256=A3xoPEBTG8Y1GMrts1ehhxt6jlbECr5UoNzko6uGFis,14432
+airbyte_cdk/sources/streams/concurrent/adapters.py,sha256=yYpmVHwRkanyz1Pfm1dbZt_Q93pGnY8cmVKDLwiFTBM,17325
 airbyte_cdk/sources/streams/concurrent/availability_strategy.py,sha256=8xDRpfktnARBbRi_RwznvKuoGrpPF2b6tQyloMwogkM,2013
+airbyte_cdk/sources/streams/concurrent/cursor.py,sha256=z9xuKGHadIOvVA67_SzlmfkQgkaudO2Te2k8Gi9OibY,6374
 airbyte_cdk/sources/streams/concurrent/exceptions.py,sha256=-WETGIY5_QFmVeDFiqm4WhRJ_nNCkfcDwOQqx6cSqrI,365
-airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py,sha256=DFDgZD_SKOcRwm634BOp2wMW37S0S1Z3JcNvJFXnl38,1682
+airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py,sha256=Xg0yeH2-aYTOvSXZrDbzGiacNr4_ARBpSjxkzdKvGoU,1602
 airbyte_cdk/sources/streams/concurrent/partition_reader.py,sha256=H8sGVVGx6uKMSUehRaqmVbE19DE3cx3NivQ4sFj8wbk,1303
-airbyte_cdk/sources/streams/concurrent/thread_based_concurrent_stream.py,sha256=Qk4zs-Zho4wBqCvVtK6stw8Y79dKFmKA8FpuEkQDqyg,9811
+airbyte_cdk/sources/streams/concurrent/thread_based_concurrent_stream.py,sha256=7DRmqrnWVWwout-MR6hBb1bIP5h5H9tU8pzXkYm3EKA,10024
 airbyte_cdk/sources/streams/concurrent/partitions/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-airbyte_cdk/sources/streams/concurrent/partitions/partition.py,sha256=r0vl7IgoOnsvWNk3_-VtDWjLYm_BPQ7tmpYmXgXLuGA,977
-airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py,sha256=6ZLDxm2LBKmC5HD_6wpgFAEH12CkOr1Mn3azIGwYfdM,537
+airbyte_cdk/sources/streams/concurrent/partitions/partition.py,sha256=tjXF8lZMvyfZaCYCHr5aTPwbVstmRjYZDwYAvLDY-ds,1312
+airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py,sha256=_ymkkBr71_qt1fW0_MUqw96OfNBkeJngXQ09yolEDHw,441
 airbyte_cdk/sources/streams/concurrent/partitions/record.py,sha256=c87pzwl18pq1_3XLoKDXH_WwrskjbBnTGkxrF4uU5-A,469
 airbyte_cdk/sources/streams/concurrent/partitions/types.py,sha256=uc3aBg2kbp3mZry3RtmAwtFExKG2oQw2qG12tZWY514,849
 airbyte_cdk/sources/streams/http/__init__.py,sha256=cTP2d7Wf0hYXaN20U0dtxKa1pFZ9rI-lBbkQ0UM1apQ,261
@@ -250,7 +251,7 @@ unit_tests/singer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 unit_tests/singer/test_singer_helpers.py,sha256=pZV6VxJuK-3-FICNGmoGbokrA_zkaFZEd4rYZCVpSRU,1762
 unit_tests/singer/test_singer_source.py,sha256=edN_kv7dnYAdBveWdUYOs74ak0dK6p8uaX225h_ZILA,4442
 unit_tests/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unit_tests/sources/test_abstract_source.py,sha256=6oo0lSfy4oKsagj-8KXGDQB8fuMvN1j9n7bXlAGCIsE,47759
+unit_tests/sources/test_abstract_source.py,sha256=V7zSpOk-jqfOz8FtnImAo_zDe-Q2TjPqD_l_T0QaiDw,48179
 unit_tests/sources/test_config.py,sha256=lxjeaf48pOMF4Pf3-Z1ux_tHTyjRFCdG_hpnxw3e7uQ,2839
 unit_tests/sources/test_connector_state_manager.py,sha256=ynFxA63Cxe6t-wMMh9C6ByTlMAuk8W7H2FikDhnUEQ0,24264
 unit_tests/sources/test_http_logger.py,sha256=VT6DqgspI3DcRnoBQkkQX0z4dF_AOiYZ5P_zxmMW8oU,9004
@@ -370,22 +371,23 @@ unit_tests/sources/file_based/stream/test_default_file_based_stream.py,sha256=An
 unit_tests/sources/fixtures/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 unit_tests/sources/fixtures/source_test_fixture.py,sha256=dvpISgio2sOp-U3bXudH_49vY4c68sO_PMs1JZTMaj0,5502
 unit_tests/sources/message/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unit_tests/sources/message/test_repository.py,sha256=zORo6lLj4L6QPpvUX4JW9XyiLSJhouFKmIzEX4Brm4M,7197
+unit_tests/sources/message/test_repository.py,sha256=oiScwg4cAdnYDl7PPN1nZniDGpALz9JTPaRAuU2HD7g,6916
 unit_tests/sources/streams/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/streams/test_availability_strategy.py,sha256=vJrSEk9NwRghu0YsSNoMYHKWzA9UFemwyClpke8Mk2s,2315
-unit_tests/sources/streams/test_stream_read.py,sha256=yzVfbVdHR7jc-_4-BfA8ySNogdYqapnUayBWuPjg1j0,6768
+unit_tests/sources/streams/test_stream_read.py,sha256=so1M-vf8HxSGq-MVrMymtCvK0abdaT_X9AfV9i0uVm0,6879
 unit_tests/sources/streams/test_streams_core.py,sha256=YOC7XqWFJ13Z4YuO9Nh4AR4AwpJ-s111vqPplFfpxk4,5059
 unit_tests/sources/streams/concurrent/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-unit_tests/sources/streams/concurrent/test_adapters.py,sha256=gTfNEDV9ZSiH9vzyuXJRQN2fvBdwZECVyczw_B8tvPc,13027
-unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py,sha256=HHaSM8PoqW-46sIjW-v3NE9iQyIKgjwcQ9nscjwsiko,1260
+unit_tests/sources/streams/concurrent/test_adapters.py,sha256=v8_tv2GCUk73DyoiPEPFDBrTRCsXysR-Cw-DXkQnPn4,14743
+unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py,sha256=6ai_6AeRuiUFB0p5TcFGHMG2eiGFbGrmXhI41Oe5XYQ,1321
+unit_tests/sources/streams/concurrent/test_cursor.py,sha256=vtKTMR4DGDKftaSny6ioKohX9bnT-mQsg_YGwkvlvwE,5107
 unit_tests/sources/streams/concurrent/test_partition_reader.py,sha256=eM5dzfmLKm9Lj-BfQUjAZRhCZzfvhk7AkKpcHGcoPfg,931
-unit_tests/sources/streams/concurrent/test_thread_based_concurrent_stream.py,sha256=XM8KKOLp-yZcU2rwd8qUXKGXXxbmAaIl_P6zzxwprG0,10048
+unit_tests/sources/streams/concurrent/test_thread_based_concurrent_stream.py,sha256=TRWhtcJ56kIgMPRST46YCERC_DVRPkAbAzVVDZgX2bI,10285
 unit_tests/sources/streams/concurrent/scenarios/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=8Wb4a0XglscnmeCyDHPGTwlX3y7mQU57EAk0q7AIoOg,2442
-unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py,sha256=_1lu7H8EdE7edC7XunhKMsGx6V9KzBCkyEF0B-hRSdI,11851
-unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py,sha256=C6rADQ3DwfwyGYP463hd46aBlJ7C3TQ8Bl6CyAvGSVc,2601
+unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=XLPC5Wqr-aBIc1tPahLKBnPHbHFw5q9rWkWCewmMeN8,4080
+unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py,sha256=XqAvndFw4i3s0Kky1AFy_asA6VDUVSj-se6rEqwf94s,15820
+unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py,sha256=yIIyEHn2jLkgryHTq9K5TSRTMit0ynIhjhMj0hKFItM,2961
 unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py,sha256=kiZ6VvQywg073FtrpP9AD3yIwSKbalVyfOM2Ksu3sZI,13554
-unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py,sha256=mnI799feDVAkDMjG-Qru8irUQECh6P2tMA9Kk-DHrW0,4732
+unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py,sha256=G4Em5zfAd9ExzDaD-86nabxWHj9wn5HT3Mfz37UNiME,5310
 unit_tests/sources/streams/http/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/streams/http/test_availability_strategy.py,sha256=kuQJ5FIc4lffpHmEUVzvoN1QXQzvz8WEkFvzHItiipg,6063
 unit_tests/sources/streams/http/test_http.py,sha256=Zo5B_PPuyoCd6Cv3q8ISCxUQCwM3rFqMM-CS6TqMvRs,24746
@@ -401,8 +403,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
 unit_tests/utils/test_stream_status_utils.py,sha256=N2TxwKge45RHUKFlPcP2o5jXYjJPKMKiu6Fm2_leZYY,3388
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.52.7.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.52.7.dist-info/METADATA,sha256=t9RNmgqP5HbmlaCqOyOXT2-l7UGke73oMmP_4QUmWa0,11945
-airbyte_cdk-0.52.7.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-airbyte_cdk-0.52.7.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.52.7.dist-info/RECORD,,
+airbyte_cdk-0.52.8.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.52.8.dist-info/METADATA,sha256=dLyy4YLIXG8SQk9Z07o05xWLhnRBCIaqC52tFodjflg,11945
+airbyte_cdk-0.52.8.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
+airbyte_cdk-0.52.8.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.52.8.dist-info/RECORD,,

unit_tests/sources/message/test_repository.py

@@ -5,15 +5,7 @@
 from unittest.mock import Mock
 
 import pytest
-from airbyte_cdk.models import (
-    AirbyteControlConnectorConfigMessage,
-    AirbyteControlMessage,
-    AirbyteMessage,
-    AirbyteStateMessage,
-    Level,
-    OrchestratorType,
-    Type,
-)
+from airbyte_cdk.models import AirbyteControlConnectorConfigMessage, AirbyteControlMessage, AirbyteMessage, Level, OrchestratorType, Type
 from airbyte_cdk.sources.message import (
     InMemoryMessageRepository,
     LogAppenderMessageRepositoryDecorator,
@@ -74,14 +66,9 @@ class TestInMemoryMessageRepository:
         second_message_generator = repo.consume_queue()
         assert list(second_message_generator) == [second_message]
 
-    def test_given_message_is_not_control_nor_log_message_when_emit_message_then_raise_error(self):
-        repo = InMemoryMessageRepository()
-        with pytest.raises(ValueError):
-            repo.emit_message(AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": "state value"})))
-
     def test_given_log_level_is_severe_enough_when_log_message_then_allow_message_to_be_consumed(self):
         repo = InMemoryMessageRepository(Level.DEBUG)
-        repo.log_message(Level.INFO, lambda: "this is a log message")
+        repo.log_message(Level.INFO, lambda: {"message": "this is a log message"})
         assert list(repo.consume_queue())
 
     def test_given_log_level_is_severe_enough_when_log_message_then_filter_secrets(self, mocker):
@@ -89,18 +76,18 @@ class TestInMemoryMessageRepository:
         mocker.patch("airbyte_cdk.sources.message.repository.filter_secrets", return_value=filtered_message)
         repo = InMemoryMessageRepository(Level.DEBUG)
 
-        repo.log_message(Level.INFO, lambda: "this is a log message")
+        repo.log_message(Level.INFO, lambda: {"message": "this is a log message"})
 
         assert list(repo.consume_queue())[0].log.message == filtered_message
 
     def test_given_log_level_not_severe_enough_when_log_message_then_do_not_allow_message_to_be_consumed(self):
         repo = InMemoryMessageRepository(Level.ERROR)
-        repo.log_message(Level.INFO, lambda: "this is a log message")
+        repo.log_message(Level.INFO, lambda: {"message": "this is a log message"})
         assert not list(repo.consume_queue())
 
     def test_given_unknown_log_level_as_threshold_when_log_message_then_allow_message_to_be_consumed(self):
         repo = InMemoryMessageRepository(UNKNOWN_LEVEL)
-        repo.log_message(Level.DEBUG, lambda: "this is a log message")
+        repo.log_message(Level.DEBUG, lambda: {"message": "this is a log message"})
         assert list(repo.consume_queue())
 
     def test_given_unknown_log_level_for_log_when_log_message_then_raise_error(self):
@@ -109,14 +96,14 @@ class TestInMemoryMessageRepository:
         """
         repo = InMemoryMessageRepository(Level.ERROR)
         with pytest.raises(ValidationError):
-            repo.log_message(UNKNOWN_LEVEL, lambda: "this is a log message")
+            repo.log_message(UNKNOWN_LEVEL, lambda: {"message": "this is a log message"})
 
 
 class TestNoopMessageRepository:
     def test_given_message_emitted_when_consume_queue_then_return_empty(self):
         repo = NoopMessageRepository()
         repo.emit_message(AirbyteMessage(type=Type.CONTROL, control=A_CONTROL))
-        repo.log_message(Level.INFO, lambda: "this is a log message")
+        repo.log_message(Level.INFO, lambda: {"message": "this is a log message"})
 
         assert not list(repo.consume_queue())
 

unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py

@@ -6,27 +6,61 @@ from typing import Any, List, Mapping, Optional, Tuple, Union
 
 from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConnectorSpecification, DestinationSyncMode, SyncMode
 from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade
+from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, NoopCursor
 from airbyte_protocol.models import ConfiguredAirbyteStream
 from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder
 
+_NO_STATE = None
+
 
 class StreamFacadeSource(AbstractSource):
-    def __init__(self, streams: List[Stream], max_workers: int):
+    def __init__(
+        self,
+        streams: List[Stream],
+        max_workers: int,
+        cursor_field: Optional[CursorField] = None,
+        cursor_boundaries: Optional[Tuple[str, str]] = None,
+    ):
         self._streams = streams
         self._max_workers = max_workers
+        self._message_repository = InMemoryMessageRepository()
+        self._cursor_field = cursor_field
+        self._cursor_boundaries = cursor_boundaries
 
     def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
         return True, None
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        return [StreamFacade.create_from_stream(stream, self, stream.logger, self._max_workers) for stream in self._streams]
+        state_manager = ConnectorStateManager(stream_instance_map={s.name: s for s in self._streams}, state=_NO_STATE)
+        return [
+            StreamFacade.create_from_stream(
+                stream,
+                self,
+                stream.logger,
+                self._max_workers,
+                _NO_STATE,
+                ConcurrentCursor(
+                    stream.name,
+                    stream.namespace,
+                    _NO_STATE,
+                    self.message_repository,  # type: ignore  # for this source specifically, we always return `InMemoryMessageRepository`
+                    state_manager,
+                    self._cursor_field,
+                    self._cursor_boundaries,
+                )
+                if self._cursor_field
+                else NoopCursor(),
+            )
+            for stream in self._streams
+        ]
 
     @property
     def message_repository(self) -> Union[None, MessageRepository]:
-        return InMemoryMessageRepository()
+        return self._message_repository
 
     def spec(self, logger: logging.Logger) -> ConnectorSpecification:
         return ConnectorSpecification(connectionSpecification={})
@@ -49,14 +83,21 @@ class StreamFacadeSourceBuilder(SourceBuilder[StreamFacadeSource]):
         self._source = None
         self._streams = []
         self._max_workers = 1
+        self._cursor_field = None
+        self._cursor_boundaries = None
 
     def set_streams(self, streams: List[Stream]) -> "StreamFacadeSourceBuilder":
         self._streams = streams
         return self
 
-    def set_max_workers(self, max_workers: int):
+    def set_max_workers(self, max_workers: int) -> "StreamFacadeSourceBuilder":
         self._max_workers = max_workers
         return self
 
+    def set_incremental(self, cursor_field: CursorField, cursor_boundaries: Optional[Tuple[str, str]]) -> "StreamFacadeSourceBuilder":
+        self._cursor_field = cursor_field
+        self._cursor_boundaries = cursor_boundaries
+        return self
+
     def build(self, configured_catalog: Optional[Mapping[str, Any]]) -> StreamFacadeSource:
-        return StreamFacadeSource(self._streams, self._max_workers)
+        return StreamFacadeSource(self._streams, self._max_workers, self._cursor_field, self._cursor_boundaries)

unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py

@@ -1,26 +1,25 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-from typing import Any, Iterable, List, Mapping, Optional, Union
+from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union
 
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
 from airbyte_cdk.sources.streams.core import StreamData
-from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder
+from unit_tests.sources.file_based.scenarios.scenario_builder import IncrementalScenarioConfig, TestScenarioBuilder
 from unit_tests.sources.streams.concurrent.scenarios.stream_facade_builder import StreamFacadeSourceBuilder
 
 
 class _MockStream(Stream):
     def __init__(
         self,
-        slice_key,
-        slice_values_to_records_or_exception: Mapping[Optional[str], List[Union[Mapping[str, Any], Exception]]],
+        slices_and_records_or_exception: Iterable[Tuple[Optional[Mapping[str, Any]], Iterable[Union[Exception, Mapping[str, Any]]]]],
         name,
         json_schema,
         primary_key=None,
     ):
-        self._slice_key = slice_key
-        self._slice_values_to_records = slice_values_to_records_or_exception
+        self._slices_and_records_or_exception = slices_and_records_or_exception
         self._name = name
         self._json_schema = json_schema
         self._primary_key = primary_key
@@ -32,19 +31,12 @@ class _MockStream(Stream):
         stream_slice: Optional[Mapping[str, Any]] = None,
         stream_state: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[StreamData]:
-        for record_or_exception in self._get_record_or_exception_iterable(stream_slice):
-            if isinstance(record_or_exception, Exception):
-                raise record_or_exception
-            else:
-                yield record_or_exception
-
-    def _get_record_or_exception_iterable(
-        self, stream_slice: Optional[Mapping[str, Any]] = None
-    ) -> Iterable[Union[Mapping[str, Any], Exception]]:
-        if stream_slice is None:
-            return self._slice_values_to_records[None]
-        else:
-            return self._slice_values_to_records[stream_slice[self._slice_key]]
+        for _slice, records_or_exception in self._slices_and_records_or_exception:
+            if stream_slice == _slice:
+                for item in records_or_exception:
+                    if isinstance(item, Exception):
+                        raise item
+                    yield item
 
     @property
     def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]:
@@ -60,16 +52,16 @@ class _MockStream(Stream):
     def stream_slices(
         self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None
     ) -> Iterable[Optional[Mapping[str, Any]]]:
-        if self._slice_key:
-            for slice_value in self._slice_values_to_records.keys():
-                yield {self._slice_key: slice_value}
+        if self._slices_and_records_or_exception:
+            yield from [_slice for _slice, records_or_exception in self._slices_and_records_or_exception]
         else:
             yield None
 
 
 _stream1 = _MockStream(
-    None,
-    {None: [{"id": "1"}, {"id": "2"}]},
+    [
+        (None, [{"id": "1"}, {"id": "2"}]),
+    ],
     "stream1",
     json_schema={
         "type": "object",
@@ -80,8 +72,9 @@ _stream1 = _MockStream(
 )
 
 _stream_raising_exception = _MockStream(
-    None,
-    {None: [{"id": "1"}, ValueError("test exception")]},
+    [
+        (None, [{"id": "1"}, ValueError("test exception")]),
+    ],
     "stream1",
     json_schema={
         "type": "object",
@@ -92,8 +85,9 @@ _stream_raising_exception = _MockStream(
 )
 
 _stream_with_primary_key = _MockStream(
-    None,
-    {None: [{"id": "1"}, {"id": "2"}]},
+    [
+        (None, [{"id": "1"}, {"id": "2"}]),
+    ],
     "stream1",
     json_schema={
         "type": "object",
@@ -105,8 +99,9 @@ _stream_with_primary_key = _MockStream(
 )
 
 _stream2 = _MockStream(
-    None,
-    {None: [{"id": "A"}, {"id": "B"}]},
+    [
+        (None, [{"id": "A"}, {"id": "B"}]),
+    ],
     "stream2",
     json_schema={
         "type": "object",
@@ -117,8 +112,9 @@ _stream2 = _MockStream(
 )
 
 _stream_with_single_slice = _MockStream(
-    "slice_key",
-    {"s1": [{"id": "1"}, {"id": "2"}]},
+    [
+        ({"slice_key": "s1"}, [{"id": "1"}, {"id": "2"}]),
+    ],
     "stream1",
     json_schema={
         "type": "object",
@@ -129,11 +125,10 @@ _stream_with_single_slice = _MockStream(
 )
 
 _stream_with_multiple_slices = _MockStream(
-    "slice_key",
-    {
-        "s1": [{"id": "1"}, {"id": "2"}],
-        "s2": [{"id": "3"}, {"id": "4"}],
-    },
+    [
+        ({"slice_key": "s1"}, [{"id": "1"}, {"id": "2"}]),
+        ({"slice_key": "s2"}, [{"id": "3"}, {"id": "4"}]),
+    ],
     "stream1",
     json_schema={
         "type": "object",
@@ -384,3 +379,125 @@ test_stream_facade_single_stream_with_multiple_slices_with_concurrency_level_two
     )
     .build()
 )
+
+
+test_incremental_stream_with_slice_boundaries = (
+    TestScenarioBuilder()
+    .set_name("test_incremental_stream_with_slice_boundaries")
+    .set_config({})
+    .set_source_builder(
+        StreamFacadeSourceBuilder()
+        .set_streams(
+            [
+                _MockStream(
+                    [
+                        ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]),
+                        ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]),
+                    ],
+                    "stream1",
+                    json_schema={
+                        "type": "object",
+                        "properties": {
+                            "id": {"type": ["null", "string"]},
+                        },
+                    },
+                )
+            ]
+        )
+        .set_incremental(CursorField("cursor_field"), ("from", "to"))
+    )
+    .set_expected_records(
+        [
+            {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"},
+            {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"},
+            {"stream1": {"slices": [{"start": 0, "end": 1}]}},
+            {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"},
+            {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"},
+            {"stream1": {"slices": [{"start": 0, "end": 1}, {"start": 1, "end": 2}]}},
+        ]
+    )
+    .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"})
+    .set_incremental_scenario_config(
+        IncrementalScenarioConfig(
+            input_state=[],
+        )
+    )
+    .build()
+)
+
+
+_NO_SLICE_BOUNDARIES = None
+test_incremental_stream_without_slice_boundaries = (
+    TestScenarioBuilder()
+    .set_name("test_incremental_stream_without_slice_boundaries")
+    .set_config({})
+    .set_source_builder(
+        StreamFacadeSourceBuilder()
+        .set_streams(
+            [
+                _MockStream(
+                    [
+                        (None, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 3}]),
+                    ],
+                    "stream1",
+                    json_schema={
+                        "type": "object",
+                        "properties": {
+                            "id": {"type": ["null", "string"]},
+                        },
+                    },
+                )
+            ]
+        )
+        .set_incremental(CursorField("cursor_field"), _NO_SLICE_BOUNDARIES)
+    )
+    .set_expected_records(
+        [
+            {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"},
+            {"data": {"id": "2", "cursor_field": 3}, "stream": "stream1"},
+            {"stream1": {"slices": [{"start": 0, "end": 3}]}},
+        ]
+    )
+    .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"})
+    .set_incremental_scenario_config(
+        IncrementalScenarioConfig(
+            input_state=[],
+        )
+    )
+    .build()
+)
+
+test_incremental_stream_with_many_slices_but_without_slice_boundaries = (
+    TestScenarioBuilder()
+    .set_name("test_incremental_stream_with_many_slices_byt_without_slice_boundaries")
+    .set_config({})
+    .set_source_builder(
+        StreamFacadeSourceBuilder()
+        .set_streams(
+            [
+                _MockStream(
+                    [
+                        ({"parent_id": 1}, [{"id": "1", "cursor_field": 0}]),
+                        ({"parent_id": 309}, [{"id": "3", "cursor_field": 0}]),
+                    ],
+                    "stream1",
+                    json_schema={
+                        "type": "object",
+                        "properties": {
+                            "id": {"type": ["null", "string"]},
+                        },
+                    },
+                )
+            ]
+        )
+        .set_incremental(CursorField("cursor_field"), _NO_SLICE_BOUNDARIES)
+    )
+    .set_expected_read_error(ValueError, "test exception")
+    .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"})
+    .set_incremental_scenario_config(
+        IncrementalScenarioConfig(
+            input_state=[],
+        )
+    )
+    .build()
+)