airbyte-cdk 0.53.9__py3-none-any.whl → 0.55.0__py3-none-any.whl

unit_tests/sources/test_source_read.py

@@ -0,0 +1,461 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+import logging
+from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union
+from unittest.mock import Mock
+
+import freezegun
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    AirbyteStream,
+    AirbyteStreamStatus,
+    AirbyteStreamStatusTraceMessage,
+    AirbyteTraceMessage,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    DestinationSyncMode,
+    StreamDescriptor,
+    SyncMode,
+    TraceType,
+)
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource
+from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
+from airbyte_cdk.sources.message import InMemoryMessageRepository
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade
+from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor
+from airbyte_cdk.sources.streams.core import StreamData
+from airbyte_cdk.utils import AirbyteTracedException
+from unit_tests.sources.streams.concurrent.scenarios.thread_based_concurrent_stream_source_builder import NeverLogSliceLogger
+
+
+class _MockStream(Stream):
+    def __init__(self, slice_to_records: Mapping[str, List[Mapping[str, Any]]], name: str):
+        self._slice_to_records = slice_to_records
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]:
+        return None
+
+    def stream_slices(
+        self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None
+    ) -> Iterable[Optional[Mapping[str, Any]]]:
+        for partition in self._slice_to_records.keys():
+            yield {"partition": partition}
+
+    def read_records(
+        self,
+        sync_mode: SyncMode,
+        cursor_field: Optional[List[str]] = None,
+        stream_slice: Optional[Mapping[str, Any]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
+    ) -> Iterable[StreamData]:
+        for record_or_exception in self._slice_to_records[stream_slice["partition"]]:
+            if isinstance(record_or_exception, Exception):
+                raise record_or_exception
+            else:
+                yield record_or_exception
+
+    def get_json_schema(self) -> Mapping[str, Any]:
+        return {}
+
+
+class _MockSource(AbstractSource):
+    message_repository = InMemoryMessageRepository()
+
+    def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
+        pass
+
+    def set_streams(self, streams):
+        self._streams = streams
+
+    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
+        return self._streams
+
+
+class _MockConcurrentSource(ConcurrentSourceAdapter):
+    message_repository = InMemoryMessageRepository()
+
+    def __init__(self, logger):
+        concurrent_source = ConcurrentSource.create(1, 1, logger, NeverLogSliceLogger(), self.message_repository)
+        super().__init__(concurrent_source)
+
+    def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
+        pass
+
+    def set_streams(self, streams):
+        self._streams = streams
+
+    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
+        return self._streams
+
+
+@freezegun.freeze_time("2020-01-01T00:00:00")
+def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_no_exceptions_are_raised():
+    records_stream_1_partition_1 = [
+        {"id": 1, "partition": "1"},
+        {"id": 2, "partition": "1"},
+    ]
+    records_stream_1_partition_2 = [
+        {"id": 3, "partition": "2"},
+        {"id": 4, "partition": "2"},
+    ]
+    records_stream_2_partition_1 = [
+        {"id": 100, "partition": "A"},
+        {"id": 200, "partition": "A"},
+    ]
+    records_stream_2_partition_2 = [
+        {"id": 300, "partition": "B"},
+        {"id": 400, "partition": "B"},
+    ]
+    stream_1_slice_to_partition = {"1": records_stream_1_partition_1, "2": records_stream_1_partition_2}
+    stream_2_slice_to_partition = {"A": records_stream_2_partition_1, "B": records_stream_2_partition_2}
+    state = None
+    logger = _init_logger()
+
+    source, concurrent_source = _init_sources([stream_1_slice_to_partition, stream_2_slice_to_partition], state, logger)
+
+    config = {}
+    catalog = _create_configured_catalog(source._streams)
+    messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, None)
+    messages_from_concurrent_source = _read_from_source(concurrent_source, logger, config, catalog, state, None)
+
+    expected_messages = [
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.STARTED)
+                ),
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.RUNNING)
+                ),
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                stream="stream0",
+                data=records_stream_1_partition_1[0],
+                emitted_at=1577836800000,
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                stream="stream0",
+                data=records_stream_1_partition_1[1],
+                emitted_at=1577836800000,
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                stream="stream0",
+                data=records_stream_1_partition_2[0],
+                emitted_at=1577836800000,
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                stream="stream0",
+                data=records_stream_1_partition_2[1],
+                emitted_at=1577836800000,
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.COMPLETE)
+                ),
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream1"), status=AirbyteStreamStatus(AirbyteStreamStatus.STARTED)
+                ),
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream1"), status=AirbyteStreamStatus(AirbyteStreamStatus.RUNNING)
+                ),
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                stream="stream1",
+                data=records_stream_2_partition_1[0],
+                emitted_at=1577836800000,
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                stream="stream1",
+                data=records_stream_2_partition_1[1],
+                emitted_at=1577836800000,
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                stream="stream1",
+                data=records_stream_2_partition_2[0],
+                emitted_at=1577836800000,
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                stream="stream1",
+                data=records_stream_2_partition_2[1],
+                emitted_at=1577836800000,
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream1"), status=AirbyteStreamStatus(AirbyteStreamStatus.COMPLETE)
+                ),
+            ),
+        ),
+    ]
+    _verify_messages(expected_messages, messages_from_abstract_source, messages_from_concurrent_source)
+
+
+@freezegun.freeze_time("2020-01-01T00:00:00")
+def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_a_traced_exception_is_raised():
+    records = [{"id": 1, "partition": "1"}, AirbyteTracedException()]
+    stream_slice_to_partition = {"1": records}
+
+    logger = _init_logger()
+    state = None
+    source, concurrent_source = _init_sources([stream_slice_to_partition], state, logger)
+    config = {}
+    catalog = _create_configured_catalog(source._streams)
+    messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, AirbyteTracedException)
+    messages_from_concurrent_source = _read_from_source(concurrent_source, logger, config, catalog, state, AirbyteTracedException)
+
+    expected_messages = [
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.STARTED)
+                ),
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.RUNNING)
+                ),
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                stream="stream0",
+                data=records[0],
+                emitted_at=1577836800000,
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.INCOMPLETE)
+                ),
+            ),
+        ),
+    ]
+    _verify_messages(expected_messages, messages_from_abstract_source, messages_from_concurrent_source)
+
+
+@freezegun.freeze_time("2020-01-01T00:00:00")
+def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_an_exception_is_raised():
+    records = [{"id": 1, "partition": "1"}, RuntimeError()]
+    stream_slice_to_partition = {"1": records}
+    logger = _init_logger()
+
+    state = None
+
+    source, concurrent_source = _init_sources([stream_slice_to_partition], state, logger)
+    config = {}
+    catalog = _create_configured_catalog(source._streams)
+    messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, RuntimeError)
+    messages_from_concurrent_source = _read_from_source(concurrent_source, logger, config, catalog, state, RuntimeError)
+
+    expected_messages = [
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.STARTED)
+                ),
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.RUNNING)
+                ),
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                stream="stream0",
+                data=records[0],
+                emitted_at=1577836800000,
+            ),
+        ),
+        AirbyteMessage(
+            type=MessageType.TRACE,
+            trace=AirbyteTraceMessage(
+                type=TraceType.STREAM_STATUS,
+                emitted_at=1577836800000.0,
+                error=None,
+                estimate=None,
+                stream_status=AirbyteStreamStatusTraceMessage(
+                    stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.INCOMPLETE)
+                ),
+            ),
+        ),
+    ]
+    _verify_messages(expected_messages, messages_from_abstract_source, messages_from_concurrent_source)
+
+
+def _init_logger():
+    logger = Mock()
+    logger.level = logging.INFO
+    logger.isEnabledFor.return_value = False
+    return logger
+
+
+def _init_sources(stream_slice_to_partitions, state, logger):
+    source = _init_source(stream_slice_to_partitions, state, logger, _MockSource())
+    concurrent_source = _init_source(stream_slice_to_partitions, state, logger, _MockConcurrentSource(logger))
+    return source, concurrent_source
+
+
+def _init_source(stream_slice_to_partitions, state, logger, source):
+    cursor = NoopCursor()
+    streams = [
+        StreamFacade.create_from_stream(_MockStream(stream_slices, f"stream{i}"), source, logger, state, cursor)
+        for i, stream_slices in enumerate(stream_slice_to_partitions)
+    ]
+    source.set_streams(streams)
+    return source
+
+
+def _create_configured_catalog(streams):
+    return ConfiguredAirbyteCatalog(
+        streams=[
+            ConfiguredAirbyteStream(
+                stream=AirbyteStream(name=s.name, json_schema={}, supported_sync_modes=[SyncMode.full_refresh]),
+                sync_mode=SyncMode.full_refresh,
+                cursor_field=None,
+                destination_sync_mode=DestinationSyncMode.overwrite,
+            )
+            for s in streams
+        ]
+    )
+
+
+def _read_from_source(source, logger, config, catalog, state, expected_exception):
+    messages = []
+    try:
+        for m in source.read(logger, config, catalog, state):
+            messages.append(m)
+    except Exception as e:
+        if expected_exception:
+            assert isinstance(e, expected_exception)
+    return messages
+
+
+def _verify_messages(expected_messages, messages_from_abstract_source, messages_from_concurrent_source):
+    assert _compare(expected_messages, messages_from_concurrent_source)
+
+
+def _compare(s, t):
+    # Use a compare method that does not require ordering or hashing the elements
+    # We can't rely on the ordering because of the multithreading
+    # AirbyteMessage does not implement __eq__ and __hash__
+    t = list(t)
+    try:
+        for elem in s:
+            t.remove(elem)
+    except ValueError:
+        print(f"ValueError: {elem}")
+        return False
+    return not t

airbyte_cdk/sources/streams/concurrent/thread_based_concurrent_stream.py

@@ -1,221 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import concurrent
-import time
-from concurrent.futures import Future
-from functools import lru_cache
-from logging import Logger
-from queue import Queue
-from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional
-
-from airbyte_cdk.models import AirbyteStream, SyncMode
-from airbyte_cdk.sources.message import MessageRepository
-from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
-from airbyte_cdk.sources.streams.concurrent.availability_strategy import AbstractAvailabilityStrategy, StreamAvailability
-from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor
-from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer
-from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader
-from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
-from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
-from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
-from airbyte_cdk.sources.streams.concurrent.partitions.types import PARTITIONS_GENERATED_SENTINEL, PartitionCompleteSentinel, QueueItem
-from airbyte_cdk.sources.utils.slice_logger import SliceLogger
-
-
-class ThreadBasedConcurrentStream(AbstractStream):
-
-    DEFAULT_TIMEOUT_SECONDS = 900
-    DEFAULT_MAX_QUEUE_SIZE = 10_000
-    DEFAULT_SLEEP_TIME = 0.1
-
-    def __init__(
-        self,
-        partition_generator: PartitionGenerator,
-        max_workers: int,
-        name: str,
-        json_schema: Mapping[str, Any],
-        availability_strategy: AbstractAvailabilityStrategy,
-        primary_key: List[str],
-        cursor_field: Optional[str],
-        slice_logger: SliceLogger,
-        logger: Logger,
-        message_repository: MessageRepository,
-        timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
-        max_concurrent_tasks: int = DEFAULT_MAX_QUEUE_SIZE,
-        sleep_time: float = DEFAULT_SLEEP_TIME,
-        cursor: Cursor = NoopCursor(),
-        namespace: Optional[str] = None,
-    ):
-        self._stream_partition_generator = partition_generator
-        self._max_workers = max_workers
-        self._threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=self._max_workers, thread_name_prefix="workerpool")
-        self._name = name
-        self._json_schema = json_schema
-        self._availability_strategy = availability_strategy
-        self._primary_key = primary_key
-        self._cursor_field = cursor_field
-        self._slice_logger = slice_logger
-        self._logger = logger
-        self._message_repository = message_repository
-        self._timeout_seconds = timeout_seconds
-        self._max_concurrent_tasks = max_concurrent_tasks
-        self._sleep_time = sleep_time
-        self._cursor = cursor
-        self._namespace = namespace
-
-    def read(self) -> Iterable[Record]:
-        """
-        Read all data from the stream (only full-refresh is supported at the moment)
-
-        Algorithm:
-        1. Submit a future to generate the stream's partition to process.
-          - This has to be done asynchronously because we sometimes need to submit requests to the API to generate all partitions (eg for substreams).
-          - The future will add the partitions to process on a work queue.
-        2. Continuously poll work from the work queue until all partitions are generated and processed
-          - If the next work item is an Exception, stop the threadpool and raise it.
-          - If the next work item is a partition, submit a future to process it.
-            - The future will add the records to emit on the work queue.
-            - Add the partitions to the partitions_to_done dict so we know it needs to complete for the sync to succeed.
-          - If the next work item is a record, yield the record.
-          - If the next work item is PARTITIONS_GENERATED_SENTINEL, all the partitions were generated.
-          - If the next work item is a PartitionCompleteSentinel, a partition is done processing.
-            - Update the value in partitions_to_done to True so we know the partition is completed.
-        """
-        self._logger.debug(f"Processing stream slices for {self.name}")
-        futures: List[Future[Any]] = []
-        queue: Queue[QueueItem] = Queue()
-        partition_generator = PartitionEnqueuer(queue, PARTITIONS_GENERATED_SENTINEL)
-        partition_reader = PartitionReader(queue)
-
-        self._submit_task(futures, partition_generator.generate_partitions, self._stream_partition_generator)
-
-        # True -> partition is done
-        # False -> partition is not done
-        partitions_to_done: Dict[Partition, bool] = {}
-
-        finished_partitions = False
-        while record_or_partition_or_exception := queue.get(block=True, timeout=self._timeout_seconds):
-            if isinstance(record_or_partition_or_exception, Exception):
-                # An exception was raised while processing the stream
-                # Stop the threadpool and raise it
-                self._stop_and_raise_exception(record_or_partition_or_exception)
-            elif record_or_partition_or_exception == PARTITIONS_GENERATED_SENTINEL:
-                # All partitions were generated
-                finished_partitions = True
-            elif isinstance(record_or_partition_or_exception, PartitionCompleteSentinel):
-                # All records for a partition were generated
-                if record_or_partition_or_exception.partition not in partitions_to_done:
-                    raise RuntimeError(
-                        f"Received sentinel for partition {record_or_partition_or_exception.partition} that was not in partitions. This is indicative of a bug in the CDK. Please contact support.partitions:\n{partitions_to_done}"
-                    )
-                partitions_to_done[record_or_partition_or_exception.partition] = True
-                self._cursor.close_partition(record_or_partition_or_exception.partition)
-            elif isinstance(record_or_partition_or_exception, Record):
-                # Emit records
-                yield record_or_partition_or_exception
-                self._cursor.observe(record_or_partition_or_exception)
-            elif isinstance(record_or_partition_or_exception, Partition):
-                # A new partition was generated and must be processed
-                partitions_to_done[record_or_partition_or_exception] = False
-                if self._slice_logger.should_log_slice_message(self._logger):
-                    self._message_repository.emit_message(
-                        self._slice_logger.create_slice_log_message(record_or_partition_or_exception.to_slice())
-                    )
-                self._submit_task(futures, partition_reader.process_partition, record_or_partition_or_exception)
-            if finished_partitions and all(partitions_to_done.values()):
-                # All partitions were generated and process. We're done here
-                break
-
-        self._check_for_errors(futures)
-
-    def _submit_task(self, futures: List[Future[Any]], function: Callable[..., Any], *args: Any) -> None:
-        # Submit a task to the threadpool, waiting if there are too many pending tasks
-        self._wait_while_too_many_pending_futures(futures)
-        futures.append(self._threadpool.submit(function, *args))
-
-    def _wait_while_too_many_pending_futures(self, futures: List[Future[Any]]) -> None:
-        # Wait until the number of pending tasks is < self._max_concurrent_tasks
-        while True:
-            self._prune_futures(futures)
-            if len(futures) < self._max_concurrent_tasks:
-                break
-            self._logger.info("Main thread is sleeping because the task queue is full...")
-            time.sleep(self._sleep_time)
-
-    def _prune_futures(self, futures: List[Future[Any]]) -> None:
-        """
-        Take a list in input and remove the futures that are completed. If a future has an exception, it'll raise and kill the stream
-        operation.
-
-        Pruning this list safely relies on the assumptions that only the main thread can modify the list of futures.
-        """
-        if len(futures) < self._max_concurrent_tasks:
-            return
-
-        for index in reversed(range(len(futures))):
-            future = futures[index]
-            optional_exception = future.exception()
-            if optional_exception:
-                exception = RuntimeError(f"Failed reading from stream {self.name} with error: {optional_exception}")
-                self._stop_and_raise_exception(exception)
-
-            if future.done():
-                futures.pop(index)
-
-    def _check_for_errors(self, futures: List[Future[Any]]) -> None:
-        exceptions_from_futures = [f for f in [future.exception() for future in futures] if f is not None]
-        if exceptions_from_futures:
-            exception = RuntimeError(f"Failed reading from stream {self.name} with errors: {exceptions_from_futures}")
-            self._stop_and_raise_exception(exception)
-        else:
-            futures_not_done = [f for f in futures if not f.done()]
-            if futures_not_done:
-                exception = RuntimeError(f"Failed reading from stream {self.name} with futures not done: {futures_not_done}")
-                self._stop_and_raise_exception(exception)
-
-    def _stop_and_raise_exception(self, exception: BaseException) -> None:
-        self._threadpool.shutdown(wait=False, cancel_futures=True)
-        raise exception
-
-    @property
-    def name(self) -> str:
-        return self._name
-
-    def check_availability(self) -> StreamAvailability:
-        return self._availability_strategy.check_availability(self._logger)
-
-    @property
-    def cursor_field(self) -> Optional[str]:
-        return self._cursor_field
-
-    @lru_cache(maxsize=None)
-    def get_json_schema(self) -> Mapping[str, Any]:
-        return self._json_schema
-
-    def as_airbyte_stream(self) -> AirbyteStream:
-        stream = AirbyteStream(name=self.name, json_schema=dict(self._json_schema), supported_sync_modes=[SyncMode.full_refresh])
-
-        if self._namespace:
-            stream.namespace = self._namespace
-
-        if self._cursor_field:
-            stream.source_defined_cursor = True
-            stream.supported_sync_modes.append(SyncMode.incremental)
-            stream.default_cursor_field = [self._cursor_field]
-
-        keys = self._primary_key
-        if keys and len(keys) > 0:
-            stream.source_defined_primary_key = [keys]
-
-        return stream
-
-    def log_stream_sync_configuration(self) -> None:
-        self._logger.debug(
-            f"Syncing stream instance: {self.name}",
-            extra={
-                "primary_key": self._primary_key,
-                "cursor_field": self.cursor_field,
-            },
-        )