airbyte-cdk 0.68.4__py3-none-any.whl → 0.69.1__py3-none-any.whl

unit_tests/sources/file_based/scenarios/csv_scenarios.py

@@ -467,30 +467,24 @@ single_csv_scenario: TestScenario[InMemoryFilesSource] = (
     )
 ).build()
 
-multi_format_analytics_scenario: TestScenario[InMemoryFilesSource] = (
+csv_analytics_scenario: TestScenario[InMemoryFilesSource] = (
     TestScenarioBuilder[InMemoryFilesSource]()
-    .set_name("multi_format_analytics")
+    .set_name("csv_analytics")
     .set_config(
         {
             "streams": [
                 {
                     "name": "stream1",
                     "format": {"filetype": "csv"},
-                    "globs": ["file1.csv"],
+                    "globs": ["a.csv"],
                     "validation_policy": "Emit Record",
                 },
                 {
                     "name": "stream2",
                     "format": {"filetype": "csv"},
-                    "globs": ["file2.csv"],
-                    "validation_policy": "Emit Record",
-                },
-                {
-                    "name": "stream3",
-                    "format": {"filetype": "jsonl"},
-                    "globs": ["file3.jsonl"],
+                    "globs": ["b.csv"],
                     "validation_policy": "Emit Record",
-                },
+                }
             ]
         }
     )
@@ -498,17 +492,21 @@ multi_format_analytics_scenario: TestScenario[InMemoryFilesSource] = (
         FileBasedSourceBuilder()
         .set_files(
             {
-                "file1.csv": {
-                    "contents": [],
+                "a.csv": {
+                    "contents": [
+                        ("col1", "col2"),
+                        ("val11a", "val12a"),
+                        ("val21a", "val22a"),
+                    ],
                     "last_modified": "2023-06-05T03:54:07.000Z",
                 },
-                "file2.csv": {
-                    "contents": [],
-                    "last_modified": "2023-06-06T03:54:07.000Z",
-                },
-                "file3.jsonl": {
-                    "contents": [],
-                    "last_modified": "2023-06-07T03:54:07.000Z",
+                "b.csv": {
+                    "contents": [
+                        ("col1", "col2", "col3"),
+                        ("val11b", "val12b", "val13b"),
+                        ("val21b", "val22b", "val23b"),
+                    ],
+                    "last_modified": "2023-06-05T03:54:07.000Z",
                 },
             }
         )
@@ -521,7 +519,12 @@ multi_format_analytics_scenario: TestScenario[InMemoryFilesSource] = (
                     "default_cursor_field": ["_ab_source_file_last_modified"],
                     "json_schema": {
                         "type": "object",
-                        "properties": {},
+                        "properties": {
+                            "col1": {"type": ["null", "string"]},
+                            "col2": {"type": ["null", "string"]},
+                            "_ab_source_file_last_modified": {"type": "string"},
+                            "_ab_source_file_url": {"type": "string"},
+                        },
                     },
                     "name": "stream1",
                     "source_defined_cursor": True,
@@ -531,30 +534,64 @@ multi_format_analytics_scenario: TestScenario[InMemoryFilesSource] = (
                     "default_cursor_field": ["_ab_source_file_last_modified"],
                     "json_schema": {
                         "type": "object",
-                        "properties": {},
+                        "properties": {
+                            "col1": {"type": ["null", "string"]},
+                            "col2": {"type": ["null", "string"]},
+                            "col3": {"type": ["null", "string"]},
+                            "_ab_source_file_last_modified": {"type": "string"},
+                            "_ab_source_file_url": {"type": "string"},
+                        },
                     },
                     "name": "stream2",
                     "source_defined_cursor": True,
                     "supported_sync_modes": ["full_refresh", "incremental"],
-                },
-                {
-                    "default_cursor_field": ["_ab_source_file_last_modified"],
-                    "json_schema": {
-                        "type": "object",
-                        "properties": {},
-                    },
-                    "name": "stream3",
-                    "source_defined_cursor": True,
-                    "supported_sync_modes": ["full_refresh", "incremental"],
-                },
+                }
             ]
         }
     )
-    .set_expected_records([])
+    .set_expected_records([
+        {
+            "data": {
+                "col1": "val11a",
+                "col2": "val12a",
+                "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
+                "_ab_source_file_url": "a.csv",
+            },
+            "stream": "stream1",
+        },
+        {
+            "data": {
+                "col1": "val21a",
+                "col2": "val22a",
+                "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
+                "_ab_source_file_url": "a.csv",
+            },
+            "stream": "stream1",
+        },
+        {
+            "data": {
+                "col1": "val11b",
+                "col2": "val12b",
+                "col3": "val13b",
+                "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
+                "_ab_source_file_url": "b.csv",
+            },
+            "stream": "stream2",
+        },
+        {
+            "data": {
+                "col1": "val21b",
+                "col2": "val22b",
+                "col3": "val23b",
+                "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
+                "_ab_source_file_url": "b.csv",
+            },
+            "stream": "stream2",
+        },
+    ])
     .set_expected_analytics(
         [
             AirbyteAnalyticsTraceMessage(type="file-cdk-csv-stream-count", value="2"),
-            AirbyteAnalyticsTraceMessage(type="file-cdk-jsonl-stream-count", value="1"),
         ]
     )
 ).build()
@@ -1450,7 +1487,6 @@ empty_schema_inference_scenario: TestScenario[InMemoryFilesSource] = (
         }
     )
     .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value)
-    .set_expected_records([])
 ).build()
 
 schemaless_csv_scenario: TestScenario[InMemoryFilesSource] = (
@@ -3009,6 +3045,61 @@ earlier_csv_scenario: TestScenario[InMemoryFilesSource] = (
             ]
         }
     )
-    .set_expected_records([])
     .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value)
 ).build()
+
+csv_no_records_scenario: TestScenario[InMemoryFilesSource] = (
+    TestScenarioBuilder[InMemoryFilesSource]()
+    .set_name("csv_empty_no_records")
+    .set_config(
+        {
+            "streams": [
+                {
+                    "name": "stream1",
+                    "globs": ["*"],
+                    "validation_policy": "Emit Record",
+                    "input_schema": '{"col1": "boolean", "col2": "string"}',
+                    "format": {
+                        "filetype": "csv",
+                        "null_values": ["null"],
+                    },
+                }
+            ],
+            "start_date": "2023-06-04T03:54:07.000000Z",
+        }
+    )
+    .set_source_builder(
+        FileBasedSourceBuilder()
+        .set_files(
+            {
+                "a.csv": {
+                    "contents": [("col1", "col2")],  # column headers, but no data rows
+                    "last_modified": "2023-06-05T03:54:07.000Z",
+                }
+            }
+        )
+        .set_file_type("csv")
+    )
+    .set_expected_catalog(
+        {
+            "streams": [
+                {
+                    "default_cursor_field": ["_ab_source_file_last_modified"],
+                    "json_schema": {
+                        "type": "object",
+                        "properties": {
+                            "col1": {"type": "boolean"},
+                            "col2": {"type": "string"},
+                            "_ab_source_file_last_modified": {"type": "string"},
+                            "_ab_source_file_url": {"type": "string"},
+                        },
+                    },
+                    "name": "stream1",
+                    "source_defined_cursor": True,
+                    "supported_sync_modes": ["full_refresh", "incremental"],
+                }
+            ]
+        }
+    )
+    .set_expected_records([])
+).build()

unit_tests/sources/file_based/stream/concurrent/test_adapters.py

@@ -23,7 +23,7 @@ from airbyte_cdk.sources.file_based.stream.concurrent.adapters import (
     FileBasedStreamPartition,
     FileBasedStreamPartitionGenerator,
 )
-from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedNoopCursor
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedFinalStateCursor
 from airbyte_cdk.sources.message import InMemoryMessageRepository
 from airbyte_cdk.sources.streams.concurrent.cursor import Cursor
 from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage
@@ -36,7 +36,7 @@ _ANY_SYNC_MODE = SyncMode.full_refresh
 _ANY_STATE = {"state_key": "state_value"}
 _ANY_CURSOR_FIELD = ["a", "cursor", "key"]
 _STREAM_NAME = "stream"
-_ANY_CURSOR = Mock(spec=FileBasedNoopCursor)
+_ANY_CURSOR = Mock(spec=FileBasedFinalStateCursor)
 
 
 @pytest.mark.parametrize(
@@ -165,7 +165,7 @@ class StreamFacadeTest(unittest.TestCase):
             supported_sync_modes=[SyncMode.full_refresh],
         )
         self._legacy_stream = DefaultFileBasedStream(
-            cursor=FileBasedNoopCursor(MagicMock()),
+            cursor=FileBasedFinalStateCursor(stream_config=MagicMock(), stream_namespace=None, message_repository=Mock()),
             config=FileBasedStreamConfig(name="stream", format=CsvFormat()),
             catalog_schema={},
             stream_reader=MagicMock(),

unit_tests/sources/file_based/test_file_based_scenarios.py

@@ -50,6 +50,7 @@ from unit_tests.sources.file_based.scenarios.concurrent_incremental_scenarios im
     single_csv_no_input_state_scenario_concurrent,
 )
 from unit_tests.sources.file_based.scenarios.csv_scenarios import (
+    csv_analytics_scenario,
     csv_autogenerate_column_names_scenario,
     csv_custom_bool_values_scenario,
     csv_custom_delimiter_in_double_quotes_scenario,
@@ -61,6 +62,7 @@ from unit_tests.sources.file_based.scenarios.csv_scenarios import (
     csv_multi_stream_scenario,
     csv_newline_in_values_not_quoted_scenario,
     csv_newline_in_values_quoted_value_scenario,
+    csv_no_records_scenario,
     csv_single_stream_scenario,
     csv_skip_after_header_scenario,
     csv_skip_before_and_after_header_scenario,
@@ -75,7 +77,6 @@ from unit_tests.sources.file_based.scenarios.csv_scenarios import (
     invalid_csv_scenario,
     multi_csv_scenario,
     multi_csv_stream_n_file_exceeds_limit_for_inference,
-    multi_format_analytics_scenario,
     multi_stream_custom_format,
     schemaless_csv_multi_stream_scenario,
     schemaless_csv_scenario,
@@ -152,7 +153,13 @@ from unit_tests.sources.file_based.scenarios.validation_policy_scenarios import
 )
 from unit_tests.sources.file_based.test_scenarios import verify_check, verify_discover, verify_read, verify_spec
 
-discover_scenarios = [
+discover_failure_scenarios = [
+    earlier_csv_scenario,
+    empty_schema_inference_scenario,
+]
+
+discover_success_scenarios = [
+    csv_no_records_scenario,
     csv_multi_stream_scenario,
     csv_single_stream_scenario,
     invalid_csv_scenario,
@@ -176,9 +183,7 @@ discover_scenarios = [
     single_csv_file_is_skipped_if_same_modified_at_as_in_history,
     single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history,
     csv_custom_format_scenario,
-    earlier_csv_scenario,
     multi_stream_custom_format,
-    empty_schema_inference_scenario,
     single_parquet_scenario,
     multi_parquet_scenario,
     parquet_various_types_scenario,
@@ -260,12 +265,14 @@ discover_scenarios = [
     single_csv_no_input_state_scenario_concurrent,
 ]
 
-read_scenarios = discover_scenarios + [
+discover_scenarios = discover_failure_scenarios + discover_success_scenarios
+
+read_scenarios = discover_success_scenarios + [
     emit_record_scenario_multi_stream,
     emit_record_scenario_single_stream,
     skip_record_scenario_multi_stream,
     skip_record_scenario_single_stream,
-    multi_format_analytics_scenario,
+    csv_analytics_scenario,
     wait_for_rediscovery_scenario_multi_stream,
     wait_for_rediscovery_scenario_single_stream,
 ]

unit_tests/sources/file_based/test_scenarios.py

@@ -16,6 +16,7 @@ from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.file_based.stream.concurrent.cursor import AbstractConcurrentFileBasedCursor
 from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput
 from airbyte_cdk.test.entrypoint_wrapper import read as entrypoint_read
+from airbyte_cdk.utils import message_utils
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 from airbyte_protocol.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalog
 from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenario
@@ -71,7 +72,7 @@ def assert_exception(expected_exception: type[BaseException], output: Entrypoint
 
 
 def _verify_read_output(output: EntrypointOutput, scenario: TestScenario[AbstractSource]) -> None:
-    records, log_messages = output.records_and_state_messages, output.logs
+    records_and_state_messages, log_messages = output.records_and_state_messages, output.logs
     logs = [message.log for message in log_messages if message.log.level.value in scenario.log_levels]
     if scenario.expected_records is None:
         return
@@ -85,7 +86,7 @@ def _verify_read_output(output: EntrypointOutput, scenario: TestScenario[Abstrac
         ),
     )
     sorted_records = sorted(
-        filter(lambda r: r.record, records),
+        filter(lambda r: r.record, records_and_state_messages),
         key=lambda record: ",".join(
             f"{k}={v}" for k, v in sorted(record.record.data.items(), key=lambda items: (items[0], items[1])) if k != "emitted_at"
         ),
@@ -104,7 +105,9 @@ def _verify_read_output(output: EntrypointOutput, scenario: TestScenario[Abstrac
             assert actual.record.stream == expected["stream"]
 
     expected_states = list(filter(lambda e: "data" not in e, expected_records))
-    states = list(filter(lambda r: r.state, records))
+    states = list(filter(lambda r: r.state, records_and_state_messages))
+    assert len(states) > 0, "No state messages emitted. Successful syncs should emit at least one stream state."
+    _verify_state_record_counts(sorted_records, states)
 
     if hasattr(scenario.source, "cursor_cls") and issubclass(scenario.source.cursor_cls, AbstractConcurrentFileBasedCursor):
         # Only check the last state emitted because we don't know the order the others will be in.
@@ -125,8 +128,34 @@ def _verify_read_output(output: EntrypointOutput, scenario: TestScenario[Abstrac
         _verify_analytics(analytics, scenario.expected_analytics)
 
 
+def _verify_state_record_counts(records: List[AirbyteMessage], states: List[AirbyteMessage]) -> None:
+    actual_record_counts = {}
+    for record in records:
+        stream_descriptor = message_utils.get_stream_descriptor(record)
+        actual_record_counts[stream_descriptor] = actual_record_counts.get(stream_descriptor, 0) + 1
+
+    state_record_count_sums = {}
+    for state_message in states:
+        stream_descriptor = message_utils.get_stream_descriptor(state_message)
+        state_record_count_sums[stream_descriptor] = (
+            state_record_count_sums.get(stream_descriptor, 0)
+            + state_message.state.sourceStats.recordCount
+        )
+
+    for stream, actual_count in actual_record_counts.items():
+        assert state_record_count_sums.get(stream) == actual_count
+
+    # We can have extra keys in state_record_count_sums if we processed a stream and reported 0 records
+    extra_keys = state_record_count_sums.keys() - actual_record_counts.keys()
+    for stream in extra_keys:
+        assert state_record_count_sums[stream] == 0
+
+
 def _verify_analytics(analytics: List[AirbyteMessage], expected_analytics: Optional[List[AirbyteAnalyticsTraceMessage]]) -> None:
     if expected_analytics:
+        assert len(analytics) == len(
+            expected_analytics), \
+            f"Number of actual analytics messages ({len(analytics)}) did not match expected ({len(expected_analytics)})"
         for actual, expected in zip(analytics, expected_analytics):
             actual_type, actual_value = actual.trace.analytics.type, actual.trace.analytics.value
             expected_type = expected.type

unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py

@@ -21,7 +21,7 @@ from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageReposi
 from airbyte_cdk.sources.source import TState
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade
-from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, NoopCursor
+from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, FinalStateCursor
 from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter
 from airbyte_protocol.models import ConfiguredAirbyteStream
 from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder
@@ -83,7 +83,7 @@ class StreamFacadeSource(ConcurrentSourceAdapter):
                     None,
                 )
                 if self._cursor_field
-                else NoopCursor(),
+                else FinalStateCursor(stream_name=stream.name, stream_namespace=stream.namespace, message_repository=self.message_repository),
             )
             for stream, state in zip(self._streams, stream_states)
         ]

unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py

@@ -4,7 +4,7 @@
 import logging
 
 from airbyte_cdk.sources.message import InMemoryMessageRepository
-from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor
+from airbyte_cdk.sources.streams.concurrent.cursor import FinalStateCursor
 from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
 from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder
@@ -15,6 +15,8 @@ from unit_tests.sources.streams.concurrent.scenarios.thread_based_concurrent_str
     InMemoryPartitionGenerator,
 )
 
+_message_repository = InMemoryMessageRepository()
+
 _id_only_stream = DefaultStream(
     partition_generator=InMemoryPartitionGenerator(
         [InMemoryPartition("partition1", "stream1", None, [Record({"id": "1"}, "stream1"), Record({"id": "2"}, "stream1")])]
@@ -30,7 +32,7 @@ _id_only_stream = DefaultStream(
     primary_key=[],
     cursor_field=None,
     logger=logging.getLogger("test_logger"),
-    cursor=NoopCursor(),
+    cursor=FinalStateCursor(stream_name="stream1", stream_namespace=None, message_repository=_message_repository),
 )
 
 _id_only_stream_with_slice_logger = DefaultStream(
@@ -48,7 +50,7 @@ _id_only_stream_with_slice_logger = DefaultStream(
     primary_key=[],
     cursor_field=None,
     logger=logging.getLogger("test_logger"),
-    cursor=NoopCursor(),
+    cursor=FinalStateCursor(stream_name="stream1", stream_namespace=None, message_repository=_message_repository),
 )
 
 _id_only_stream_with_primary_key = DefaultStream(
@@ -66,7 +68,7 @@ _id_only_stream_with_primary_key = DefaultStream(
     primary_key=["id"],
     cursor_field=None,
     logger=logging.getLogger("test_logger"),
-    cursor=NoopCursor(),
+    cursor=FinalStateCursor(stream_name="stream1", stream_namespace=None, message_repository=_message_repository),
 )
 
 _id_only_stream_multiple_partitions = DefaultStream(
@@ -87,7 +89,7 @@ _id_only_stream_multiple_partitions = DefaultStream(
     primary_key=[],
     cursor_field=None,
     logger=logging.getLogger("test_logger"),
-    cursor=NoopCursor(),
+    cursor=FinalStateCursor(stream_name="stream1", stream_namespace=None, message_repository=_message_repository),
 )
 
 _id_only_stream_multiple_partitions_concurrency_level_two = DefaultStream(
@@ -108,7 +110,7 @@ _id_only_stream_multiple_partitions_concurrency_level_two = DefaultStream(
     primary_key=[],
     cursor_field=None,
     logger=logging.getLogger("test_logger"),
-    cursor=NoopCursor(),
+    cursor=FinalStateCursor(stream_name="stream1", stream_namespace=None, message_repository=_message_repository),
 )
 
 _stream_raising_exception = DefaultStream(
@@ -126,7 +128,7 @@ _stream_raising_exception = DefaultStream(
     primary_key=[],
     cursor_field=None,
     logger=logging.getLogger("test_logger"),
-    cursor=NoopCursor(),
+    cursor=FinalStateCursor(stream_name="stream1", stream_namespace=None, message_repository=_message_repository),
 )
 
 test_concurrent_cdk_single_stream = (
@@ -140,7 +142,7 @@ test_concurrent_cdk_single_stream = (
                 _id_only_stream,
             ]
         )
-        .set_message_repository(InMemoryMessageRepository())
+        .set_message_repository(_message_repository)
     )
     .set_expected_records(
         [
@@ -193,7 +195,7 @@ test_concurrent_cdk_single_stream_with_primary_key = (
                 _id_only_stream_with_primary_key,
             ]
         )
-        .set_message_repository(InMemoryMessageRepository())
+        .set_message_repository(_message_repository)
     )
     .set_expected_records(
         [
@@ -253,11 +255,11 @@ test_concurrent_cdk_multiple_streams = (
                     primary_key=[],
                     cursor_field=None,
                     logger=logging.getLogger("test_logger"),
-                    cursor=NoopCursor(),
+                    cursor=FinalStateCursor(stream_name="stream2", stream_namespace=None, message_repository=_message_repository),
                 ),
             ]
         )
-        .set_message_repository(InMemoryMessageRepository())
+        .set_message_repository(_message_repository)
     )
     .set_expected_records(
         [
@@ -308,7 +310,7 @@ test_concurrent_cdk_partition_raises_exception = (
                 _stream_raising_exception,
             ]
         )
-        .set_message_repository(InMemoryMessageRepository())
+        .set_message_repository(_message_repository)
     )
     .set_expected_records(
         [
@@ -346,7 +348,7 @@ test_concurrent_cdk_single_stream_multiple_partitions = (
                 _id_only_stream_multiple_partitions,
             ]
         )
-        .set_message_repository(InMemoryMessageRepository())
+        .set_message_repository(_message_repository)
     )
     .set_expected_records(
         [
@@ -386,7 +388,7 @@ test_concurrent_cdk_single_stream_multiple_partitions_concurrency_level_two = (
                 _id_only_stream_multiple_partitions_concurrency_level_two,
             ]
         )
-        .set_message_repository(InMemoryMessageRepository())
+        .set_message_repository(_message_repository)
     )
     .set_expected_records(
         [

unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py

@@ -8,11 +8,11 @@ from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union
 from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConnectorSpecification, DestinationSyncMode, SyncMode
 from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource
 from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
-from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade
 from airbyte_cdk.sources.streams.concurrent.availability_strategy import AbstractAvailabilityStrategy, StreamAvailability, StreamAvailable
-from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor
+from airbyte_cdk.sources.streams.concurrent.cursor import FinalStateCursor
 from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
 from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
@@ -42,13 +42,14 @@ class ConcurrentCdkSource(ConcurrentSourceAdapter):
         concurrent_source = ConcurrentSource.create(1, 1, streams[0]._logger, NeverLogSliceLogger(), message_repository)
         super().__init__(concurrent_source)
         self._streams = streams
+        self._message_repository = message_repository
 
     def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
         # Check is not verified because it is up to the source to implement this method
         return True, None
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        return [StreamFacade(s, LegacyStream(), NoopCursor(), NeverLogSliceLogger(), s._logger) for s in self._streams]
+        return [StreamFacade(s, LegacyStream(), FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=self.message_repository), NeverLogSliceLogger(), s._logger) for s in self._streams]
 
     def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification:
         return ConnectorSpecification(connectionSpecification={})
@@ -57,7 +58,7 @@ class ConcurrentCdkSource(ConcurrentSourceAdapter):
         return ConfiguredAirbyteCatalog(
             streams=[
                 ConfiguredAirbyteStream(
-                    stream=StreamFacade(s, LegacyStream(), NoopCursor(), NeverLogSliceLogger(), s._logger).as_airbyte_stream(),
+                    stream=StreamFacade(s, LegacyStream(), FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=InMemoryMessageRepository()), NeverLogSliceLogger(), s._logger).as_airbyte_stream(),
                     sync_mode=SyncMode.full_refresh,
                     destination_sync_mode=DestinationSyncMode.overwrite,
                 )

unit_tests/sources/streams/concurrent/test_default_stream.py

@@ -5,8 +5,9 @@ import unittest
 from unittest.mock import Mock
 
 from airbyte_cdk.models import AirbyteStream, SyncMode
+from airbyte_cdk.sources.message import InMemoryMessageRepository
 from airbyte_cdk.sources.streams.concurrent.availability_strategy import STREAM_AVAILABLE
-from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor
+from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, FinalStateCursor
 from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 
 
@@ -20,6 +21,7 @@ class ThreadBasedConcurrentStreamTest(unittest.TestCase):
         self._cursor_field = None
         self._logger = Mock()
         self._cursor = Mock(spec=Cursor)
+        self._message_repository = InMemoryMessageRepository()
         self._stream = DefaultStream(
             self._partition_generator,
             self._name,
@@ -28,7 +30,7 @@ class ThreadBasedConcurrentStreamTest(unittest.TestCase):
             self._primary_key,
             self._cursor_field,
             self._logger,
-            NoopCursor(),
+            FinalStateCursor(stream_name=self._name, stream_namespace=None, message_repository=self._message_repository),
         )
 
     def test_get_json_schema(self):
@@ -89,7 +91,7 @@ class ThreadBasedConcurrentStreamTest(unittest.TestCase):
             ["id"],
             self._cursor_field,
             self._logger,
-            NoopCursor(),
+            FinalStateCursor(stream_name=self._name, stream_namespace=None, message_repository=self._message_repository),
         )
 
         expected_airbyte_stream = AirbyteStream(
@@ -121,7 +123,7 @@ class ThreadBasedConcurrentStreamTest(unittest.TestCase):
             ["id_a", "id_b"],
             self._cursor_field,
             self._logger,
-            NoopCursor(),
+            FinalStateCursor(stream_name=self._name, stream_namespace=None, message_repository=self._message_repository),
         )
 
         expected_airbyte_stream = AirbyteStream(
@@ -153,7 +155,7 @@ class ThreadBasedConcurrentStreamTest(unittest.TestCase):
             self._primary_key,
             "date",
             self._logger,
-            NoopCursor(),
+            FinalStateCursor(stream_name=self._name, stream_namespace=None, message_repository=self._message_repository),
         )
 
         expected_airbyte_stream = AirbyteStream(
@@ -178,7 +180,7 @@ class ThreadBasedConcurrentStreamTest(unittest.TestCase):
             self._primary_key,
             self._cursor_field,
             self._logger,
-            NoopCursor(),
+            FinalStateCursor(stream_name=self._name, stream_namespace=None, message_repository=self._message_repository),
             namespace="test",
         )
         expected_airbyte_stream = AirbyteStream(

unit_tests/sources/streams/test_stream_read.py

@@ -26,7 +26,7 @@ from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade
-from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor
+from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, FinalStateCursor
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
 from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
 from airbyte_cdk.sources.streams.core import StreamData
@@ -105,8 +105,9 @@ def _stream(slice_to_partition_mapping, slice_logger, logger, message_repository
     return _MockStream(slice_to_partition_mapping)
 
 
-def _concurrent_stream(slice_to_partition_mapping, slice_logger, logger, message_repository, cursor: Cursor = NoopCursor()):
+def _concurrent_stream(slice_to_partition_mapping, slice_logger, logger, message_repository, cursor: Optional[Cursor] = None):
     stream = _stream(slice_to_partition_mapping, slice_logger, logger, message_repository)
+    cursor = cursor or FinalStateCursor(stream_name=stream.name, stream_namespace=stream.namespace, message_repository=message_repository)
     source = Mock()
     source._slice_logger = slice_logger
     source.message_repository = message_repository