airbyte-cdk 0.54.0__py3-none-any.whl → 0.55.0__py3-none-any.whl

airbyte_cdk/sources/streams/concurrent/adapters.py

@@ -22,11 +22,11 @@ from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
     StreamUnavailable,
 )
 from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor
+from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
 from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
 from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
-from airbyte_cdk.sources.streams.concurrent.thread_based_concurrent_stream import ThreadBasedConcurrentStream
 from airbyte_cdk.sources.streams.core import StreamData
 from airbyte_cdk.sources.utils.schema_helpers import InternalConfig
 from airbyte_cdk.sources.utils.slice_logger import SliceLogger
@@ -52,7 +52,6 @@ class StreamFacade(Stream):
         stream: Stream,
         source: AbstractSource,
         logger: logging.Logger,
-        max_workers: int,
         state: Optional[MutableMapping[str, Any]],
         cursor: Cursor,
     ) -> Stream:
@@ -73,28 +72,27 @@ class StreamFacade(Stream):
 
         message_repository = source.message_repository
         return StreamFacade(
-            ThreadBasedConcurrentStream(
+            DefaultStream(
                 partition_generator=StreamPartitionGenerator(
                     stream,
                     message_repository,
                     SyncMode.full_refresh if isinstance(cursor, NoopCursor) else SyncMode.incremental,
                     [cursor_field] if cursor_field is not None else None,
                     state,
+                    cursor,
                 ),
-                max_workers=max_workers,
                 name=stream.name,
                 namespace=stream.namespace,
                 json_schema=stream.get_json_schema(),
                 availability_strategy=StreamAvailabilityStrategy(stream, source),
                 primary_key=pk,
                 cursor_field=cursor_field,
-                slice_logger=source._slice_logger,
-                message_repository=message_repository,
                 logger=logger,
-                cursor=cursor,
             ),
             stream,
             cursor,
+            slice_logger=source._slice_logger,
+            logger=logger,
         )
 
     @property
@@ -132,13 +130,15 @@ class StreamFacade(Stream):
         else:
             return stream.cursor_field
 
-    def __init__(self, stream: AbstractStream, legacy_stream: Stream, cursor: Cursor):
+    def __init__(self, stream: AbstractStream, legacy_stream: Stream, cursor: Cursor, slice_logger: SliceLogger, logger: logging.Logger):
         """
         :param stream: The underlying AbstractStream
         """
         self._abstract_stream = stream
         self._legacy_stream = legacy_stream
         self._cursor = cursor
+        self._slice_logger = slice_logger
+        self._logger = logger
 
     def read_full_refresh(
         self,
@@ -177,8 +177,11 @@ class StreamFacade(Stream):
         yield from self._read_records()
 
     def _read_records(self) -> Iterable[StreamData]:
-        for record in self._abstract_stream.read():
-            yield record.data
+        for partition in self._abstract_stream.generate_partitions():
+            if self._slice_logger.should_log_slice_message(self._logger):
+                yield self._slice_logger.create_slice_log_message(partition.to_slice())
+            for record in partition.read():
+                yield record.data
 
     @property
     def name(self) -> str:
@@ -259,6 +262,7 @@ class StreamPartition(Partition):
         sync_mode: SyncMode,
         cursor_field: Optional[List[str]],
         state: Optional[MutableMapping[str, Any]],
+        cursor: Cursor,
     ):
         """
         :param stream: The stream to delegate to
@@ -271,6 +275,8 @@ class StreamPartition(Partition):
         self._sync_mode = sync_mode
         self._cursor_field = cursor_field
         self._state = state
+        self._cursor = cursor
+        self._is_closed = False
 
     def read(self) -> Iterable[Record]:
         """
@@ -294,7 +300,9 @@ class StreamPartition(Partition):
                 if isinstance(record_data, Mapping):
                     data_to_return = dict(record_data)
                     self._stream.transformer.transform(data_to_return, self._stream.get_json_schema())
-                    yield Record(data_to_return)
+                    record = Record(data_to_return, self._stream.name)
+                    self._cursor.observe(record)
+                    yield Record(data_to_return, self._stream.name)
                 else:
                     self._message_repository.emit_message(record_data)
         except Exception as e:
@@ -315,6 +323,16 @@ class StreamPartition(Partition):
         else:
             return hash(self._stream.name)
 
+    def stream_name(self) -> str:
+        return self._stream.name
+
+    def close(self) -> None:
+        self._cursor.close_partition(self)
+        self._is_closed = True
+
+    def is_closed(self) -> bool:
+        return self._is_closed
+
     def __repr__(self) -> str:
         return f"StreamPartition({self._stream.name}, {self._slice})"
 
@@ -334,6 +352,7 @@ class StreamPartitionGenerator(PartitionGenerator):
         sync_mode: SyncMode,
         cursor_field: Optional[List[str]],
         state: Optional[MutableMapping[str, Any]],
+        cursor: Cursor,
     ):
         """
         :param stream: The stream to delegate to
@@ -344,10 +363,13 @@ class StreamPartitionGenerator(PartitionGenerator):
         self._sync_mode = sync_mode
         self._cursor_field = cursor_field
         self._state = state
+        self._cursor = cursor
 
     def generate(self) -> Iterable[Partition]:
         for s in self._stream.stream_slices(sync_mode=self._sync_mode, cursor_field=self._cursor_field, stream_state=self._state):
-            yield StreamPartition(self._stream, copy.deepcopy(s), self.message_repository, self._sync_mode, self._cursor_field, self._state)
+            yield StreamPartition(
+                self._stream, copy.deepcopy(s), self.message_repository, self._sync_mode, self._cursor_field, self._state, self._cursor
+            )
 
 
 @deprecated("This class is experimental. Use at your own risk.")

airbyte_cdk/sources/streams/concurrent/default_stream.py

@@ -0,0 +1,79 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from functools import lru_cache
+from logging import Logger
+from typing import Any, Iterable, List, Mapping, Optional
+
+from airbyte_cdk.models import AirbyteStream, SyncMode
+from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
+from airbyte_cdk.sources.streams.concurrent.availability_strategy import AbstractAvailabilityStrategy, StreamAvailability
+from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
+from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
+
+
+class DefaultStream(AbstractStream):
+    def __init__(
+        self,
+        partition_generator: PartitionGenerator,
+        name: str,
+        json_schema: Mapping[str, Any],
+        availability_strategy: AbstractAvailabilityStrategy,
+        primary_key: List[str],
+        cursor_field: Optional[str],
+        logger: Logger,
+        namespace: Optional[str] = None,
+    ) -> None:
+        self._stream_partition_generator = partition_generator
+        self._name = name
+        self._json_schema = json_schema
+        self._availability_strategy = availability_strategy
+        self._primary_key = primary_key
+        self._cursor_field = cursor_field
+        self._logger = logger
+        self._namespace = namespace
+
+    def generate_partitions(self) -> Iterable[Partition]:
+        yield from self._stream_partition_generator.generate()
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def check_availability(self) -> StreamAvailability:
+        return self._availability_strategy.check_availability(self._logger)
+
+    @property
+    def cursor_field(self) -> Optional[str]:
+        return self._cursor_field
+
+    @lru_cache(maxsize=None)
+    def get_json_schema(self) -> Mapping[str, Any]:
+        return self._json_schema
+
+    def as_airbyte_stream(self) -> AirbyteStream:
+        stream = AirbyteStream(name=self.name, json_schema=dict(self._json_schema), supported_sync_modes=[SyncMode.full_refresh])
+
+        if self._namespace:
+            stream.namespace = self._namespace
+
+        if self._cursor_field:
+            stream.source_defined_cursor = True
+            stream.supported_sync_modes.append(SyncMode.incremental)
+            stream.default_cursor_field = [self._cursor_field]
+
+        keys = self._primary_key
+        if keys and len(keys) > 0:
+            stream.source_defined_primary_key = [keys]
+
+        return stream
+
+    def log_stream_sync_configuration(self) -> None:
+        self._logger.debug(
+            f"Syncing stream instance: {self.name}",
+            extra={
+                "primary_key": self._primary_key,
+                "cursor_field": self.cursor_field,
+            },
+        )

airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py

@@ -4,8 +4,9 @@
 
 from queue import Queue
 
-from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
-from airbyte_cdk.sources.streams.concurrent.partitions.types import PARTITIONS_GENERATED_SENTINEL, QueueItem
+from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel
+from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
+from airbyte_cdk.sources.streams.concurrent.partitions.types import QueueItem
 
 
 class PartitionEnqueuer:
@@ -13,15 +14,14 @@ class PartitionEnqueuer:
     Generates partitions from a partition generator and puts them in a queue.
     """
 
-    def __init__(self, queue: Queue[QueueItem], sentinel: PARTITIONS_GENERATED_SENTINEL) -> None:
+    def __init__(self, queue: Queue[QueueItem]) -> None:
         """
         :param queue:  The queue to put the partitions in.
         :param sentinel: The sentinel to put in the queue when all the partitions have been generated.
         """
         self._queue = queue
-        self._sentinel = sentinel
 
-    def generate_partitions(self, partition_generator: PartitionGenerator) -> None:
+    def generate_partitions(self, stream: AbstractStream) -> None:
         """
         Generate partitions from a partition generator and put them in a queue.
         When all the partitions are added to the queue, a sentinel is added to the queue to indicate that all the partitions have been generated.
@@ -33,8 +33,8 @@ class PartitionEnqueuer:
         :return:
         """
         try:
-            for partition in partition_generator.generate():
+            for partition in stream.generate_partitions():
                 self._queue.put(partition)
-            self._queue.put(self._sentinel)
+            self._queue.put(PartitionGenerationCompletedSentinel(stream))
         except Exception as e:
             self._queue.put(e)

airbyte_cdk/sources/streams/concurrent/partitions/partition.py

@@ -32,6 +32,29 @@ class Partition(ABC):
         """
         pass
 
+    @abstractmethod
+    def stream_name(self) -> str:
+        """
+        Returns the name of the stream that this partition is reading from.
+        :return: The name of the stream.
+        """
+        pass
+
+    @abstractmethod
+    def close(self) -> None:
+        """
+        Closes the partition.
+        """
+        pass
+
+    @abstractmethod
+    def is_closed(self) -> bool:
+        """
+        Returns whether the partition is closed.
+        :return:
+        """
+        pass
+
     @abstractmethod
     def __hash__(self) -> int:
         """

airbyte_cdk/sources/streams/concurrent/partitions/record.py

@@ -10,13 +10,14 @@ class Record:
     Represents a record read from a stream.
     """
 
-    def __init__(self, data: Mapping[str, Any]):
+    def __init__(self, data: Mapping[str, Any], stream_name: str):
         self.data = data
+        self.stream_name = stream_name
 
     def __eq__(self, other: Any) -> bool:
         if not isinstance(other, Record):
             return False
-        return self.data == other.data
+        return self.data == other.data and self.stream_name == other.stream_name
 
     def __repr__(self) -> str:
-        return f"Record(data={self.data})"
+        return f"Record(data={self.data}, stream_name={self.stream_name})"

airbyte_cdk/sources/streams/concurrent/partitions/types.py

@@ -4,11 +4,10 @@
 
 from typing import Union
 
+from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
 from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
 
-PARTITIONS_GENERATED_SENTINEL = object
-
 
 class PartitionCompleteSentinel:
     """
@@ -26,4 +25,4 @@ class PartitionCompleteSentinel:
 """
 Typedef representing the items that can be added to the ThreadBasedConcurrentStream
 """
-QueueItem = Union[Record, Partition, PartitionCompleteSentinel, PARTITIONS_GENERATED_SENTINEL, Partition, Exception]
+QueueItem = Union[Record, Partition, PartitionCompleteSentinel, PartitionGenerationCompletedSentinel, Exception]

airbyte_cdk/sources/utils/slice_logger.py

@@ -12,6 +12,11 @@ from airbyte_cdk.models import Type as MessageType
 
 
 class SliceLogger(ABC):
+    """
+    SliceLogger is an interface that allows us to log slices of data in a uniform way.
+    It is responsible for determining whether or not a slice should be logged and for creating the log message.
+    """
+
     SLICE_LOG_PREFIX = "slice:"
 
     def create_slice_log_message(self, _slice: Optional[Mapping[str, Any]]) -> AirbyteMessage:

{airbyte_cdk-0.54.0.dist-info → airbyte_cdk-0.55.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.54.0
+Version: 0.55.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.54.0.dist-info → airbyte_cdk-0.55.0.dist-info}/RECORD

@@ -30,6 +30,12 @@ airbyte_cdk/sources/connector_state_manager.py,sha256=wsmUgII398MazCTKxwLBLzeiU6
 airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
 airbyte_cdk/sources/http_logger.py,sha256=v0kkpDtA0GUOgj6_3AayrYaBrSHBqG4t3MGbrtxaNmU,1437
 airbyte_cdk/sources/source.py,sha256=dk50z8Roc28MJ8FxWe652B-GwItO__bTZqFm7WOtHnw,4412
+airbyte_cdk/sources/concurrent_source/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
+airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py,sha256=7A8bdOqg9Hpw37QgqoGO6eYBx8_pKikt_AJUcImp-x4,9715
+airbyte_cdk/sources/concurrent_source/concurrent_source.py,sha256=nmHZLmUwHYW5a8B-XpwVLZ2weys5oWTVYSMseW7WdYc,7836
+airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py,sha256=si5ipxvzCE2Pdusg19evr8ziG7eqBBuBuDPww6i3Amg,3223
+airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py,sha256=oExaUlnDepGZjNmauIkFDCbWtxZvkBCFo1K0wAr4sRA,493
+airbyte_cdk/sources/concurrent_source/thread_pool_manager.py,sha256=huFp0uuG2kZAnbHY8oeDYuX0hfmP-rLOnJMa72ZuWt0,3905
 airbyte_cdk/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/create_partial.py,sha256=sUJOwD8hBzW4pxw2XhYlSTMgl-WMc5WpP5Oq_jo3fHw,3371
 airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=LtLvEpzKo86RzMO6n20-z4ECW6P0Yoi26HXRCSLP9K0,85049
@@ -191,20 +197,20 @@ airbyte_cdk/sources/streams/availability_strategy.py,sha256=7BM0qLvXS0QrlKvnVkBE
 airbyte_cdk/sources/streams/call_rate.py,sha256=5T4J8WxMNov76iXRUtD5KlM1CsROxuAQPwGQAZyvpHg,20555
 airbyte_cdk/sources/streams/core.py,sha256=bIuQV7Zs9JpIyNDcfPCbyzv-BWDr_2ictK7s5AihLZQ,16025
 airbyte_cdk/sources/streams/concurrent/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-airbyte_cdk/sources/streams/concurrent/abstract_stream.py,sha256=GCcRvUixoDOkNPy0vK37xdGxYaOfZXxVH6InzndduaE,3525
-airbyte_cdk/sources/streams/concurrent/adapters.py,sha256=yYpmVHwRkanyz1Pfm1dbZt_Q93pGnY8cmVKDLwiFTBM,17325
+airbyte_cdk/sources/streams/concurrent/abstract_stream.py,sha256=W7WEz6FrfAjb0o_msnMBIESSVO1qJC2_A8ocYg55Rw4,3579
+airbyte_cdk/sources/streams/concurrent/adapters.py,sha256=kBlHLeHi_xg_rtPMzfyU4Osh-k7K5NLUMOMfAPwiwrk,18090
 airbyte_cdk/sources/streams/concurrent/availability_strategy.py,sha256=8xDRpfktnARBbRi_RwznvKuoGrpPF2b6tQyloMwogkM,2013
 airbyte_cdk/sources/streams/concurrent/cursor.py,sha256=vvQeY-IkJ8gfwKfCZTLTVlBYA9gVgQX6bYdWUT0D-q4,6504
+airbyte_cdk/sources/streams/concurrent/default_stream.py,sha256=w83pvFbw9vjfhbovw-LrCFiwQMO8hfo1Vm-1CB1SeXQ,2777
 airbyte_cdk/sources/streams/concurrent/exceptions.py,sha256=-WETGIY5_QFmVeDFiqm4WhRJ_nNCkfcDwOQqx6cSqrI,365
-airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py,sha256=uAUhCkxFOaptDJfIEDmFnnF4xn4coG5kvE4B_5tVx14,1557
+airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py,sha256=Mmn0hYq2xWe2a0WOpZPF3iZNozfmv7vY37LgfdY7DVo,1570
 airbyte_cdk/sources/streams/concurrent/partition_reader.py,sha256=H8sGVVGx6uKMSUehRaqmVbE19DE3cx3NivQ4sFj8wbk,1303
 airbyte_cdk/sources/streams/concurrent/state_converter.py,sha256=PwqcRVPR6LQxWL0yvPTp_u2Uh0hBJU-BDSjPKiyJVEk,4689
-airbyte_cdk/sources/streams/concurrent/thread_based_concurrent_stream.py,sha256=M7CpPPBswHTYjG4opiTOf5eWHOJ6i4TyP0v991pFxOo,10843
 airbyte_cdk/sources/streams/concurrent/partitions/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-airbyte_cdk/sources/streams/concurrent/partitions/partition.py,sha256=tjXF8lZMvyfZaCYCHr5aTPwbVstmRjYZDwYAvLDY-ds,1312
+airbyte_cdk/sources/streams/concurrent/partitions/partition.py,sha256=o2QvDYZF3Tn9NbC5jc1UkDwMiCWq9fNGj493u2WFoko,1795
 airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py,sha256=_ymkkBr71_qt1fW0_MUqw96OfNBkeJngXQ09yolEDHw,441
-airbyte_cdk/sources/streams/concurrent/partitions/record.py,sha256=c87pzwl18pq1_3XLoKDXH_WwrskjbBnTGkxrF4uU5-A,469
-airbyte_cdk/sources/streams/concurrent/partitions/types.py,sha256=uc3aBg2kbp3mZry3RtmAwtFExKG2oQw2qG12tZWY514,849
+airbyte_cdk/sources/streams/concurrent/partitions/record.py,sha256=-Q3zLex3CHOXiB-KOZLbBZaPiQ_BLFJdknr6yoRz9I0,600
+airbyte_cdk/sources/streams/concurrent/partitions/types.py,sha256=iVARnsGOSdvlSCqAf-yxc4_PUT3oOR9B6cyVNcLTjY8,932
 airbyte_cdk/sources/streams/http/__init__.py,sha256=cTP2d7Wf0hYXaN20U0dtxKa1pFZ9rI-lBbkQ0UM1apQ,261
 airbyte_cdk/sources/streams/http/availability_strategy.py,sha256=MHgW42gwaevaCVnNLrUSE6WJHT4reeZ417nMWrmbC7o,6884
 airbyte_cdk/sources/streams/http/exceptions.py,sha256=OokLDI7W8hZvq9e15sL3em2AdwmzmcAl72Ms-i5l0Nw,1334
@@ -227,7 +233,7 @@ airbyte_cdk/sources/utils/catalog_helpers.py,sha256=Jo3F6NQE2O7aP4x7yGScwbvtPQyC
 airbyte_cdk/sources/utils/record_helper.py,sha256=lNtOK1rMUxB9cw6wIi3yNu85jlqTN5inTkZZCWvPKXA,1711
 airbyte_cdk/sources/utils/schema_helpers.py,sha256=_Kasvdo60OE1aHkrd2Q48OHrMJnZ8nSWliuAVbR7vJs,8483
 airbyte_cdk/sources/utils/schema_models.py,sha256=m1vOqNkkVYGblc492wKo11Zm5FK9F0-JoNb50aRZnew,3151
-airbyte_cdk/sources/utils/slice_logger.py,sha256=Dv0m6Y4YYbHKB-RoMZosKjMgarQZJc4dp7jmKdwilJU,1515
+airbyte_cdk/sources/utils/slice_logger.py,sha256=YeWSoZeOsQp9oZK7mick2J8KFdiY726LY2iiIj_--r4,1731
 airbyte_cdk/sources/utils/transform.py,sha256=4GYmO6bq33HF-a1in0dKQKqUOYI1bWItyuYF875bSQg,9493
 airbyte_cdk/sources/utils/types.py,sha256=41ZQR681t5TUnOScij58d088sb99klH_ZENFcaYro_g,175
 airbyte_cdk/utils/__init__.py,sha256=qZoNqzEKhIXdN_ZfvXlIGnmiDDjCFy6BVCzzWjUZcuU,294
@@ -256,11 +262,15 @@ unit_tests/singer/test_singer_helpers.py,sha256=pZV6VxJuK-3-FICNGmoGbokrA_zkaFZE
 unit_tests/singer/test_singer_source.py,sha256=edN_kv7dnYAdBveWdUYOs74ak0dK6p8uaX225h_ZILA,4442
 unit_tests/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/test_abstract_source.py,sha256=V7zSpOk-jqfOz8FtnImAo_zDe-Q2TjPqD_l_T0QaiDw,48179
+unit_tests/sources/test_concurrent_source.py,sha256=NT4K0z-oz2OZBHE9xNQT0KUdI2wJ-5vNWLUHZlIYKKU,3552
 unit_tests/sources/test_config.py,sha256=lxjeaf48pOMF4Pf3-Z1ux_tHTyjRFCdG_hpnxw3e7uQ,2839
 unit_tests/sources/test_connector_state_manager.py,sha256=ynFxA63Cxe6t-wMMh9C6ByTlMAuk8W7H2FikDhnUEQ0,24264
 unit_tests/sources/test_http_logger.py,sha256=VT6DqgspI3DcRnoBQkkQX0z4dF_AOiYZ5P_zxmMW8oU,9004
 unit_tests/sources/test_integration_source.py,sha256=7DAWzuYwU_HXzhw-rRjjwQuQej-hVpNyzw_NLqQiJVc,3369
 unit_tests/sources/test_source.py,sha256=W0I4umL_d_OToLYYiRkjkJR6e-cCYjdV8zKc3uLvF0k,27999
+unit_tests/sources/test_source_read.py,sha256=AEFoJfzM0_5QQIJyKwGLK_kq_Vz_CBivImnUnXJQJ0I,17176
+unit_tests/sources/concurrent_source/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
+unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py,sha256=zsGnMcEsBedjW8wahil6LNqniil-3NXhyZd5W-80Km0,3665
 unit_tests/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 unit_tests/sources/declarative/external_component.py,sha256=lU2gL736bLEWtmrGm1B2k83RXt_3XkROimLIahZd5dg,293
 unit_tests/sources/declarative/test_create_partial.py,sha256=s_KIywQqt8RlauOCWNJVk3HC3KBTAtSwFTN6JVQgu80,2636
@@ -379,22 +389,24 @@ unit_tests/sources/message/test_repository.py,sha256=oiScwg4cAdnYDl7PPN1nZniDGpA
 unit_tests/sources/streams/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/streams/test_availability_strategy.py,sha256=vJrSEk9NwRghu0YsSNoMYHKWzA9UFemwyClpke8Mk2s,2315
 unit_tests/sources/streams/test_call_rate.py,sha256=5QsokqxIFoR438QTd7p_eb0K-LW6awZXDtQiMTAb_Qo,13069
-unit_tests/sources/streams/test_stream_read.py,sha256=so1M-vf8HxSGq-MVrMymtCvK0abdaT_X9AfV9i0uVm0,6879
+unit_tests/sources/streams/test_stream_read.py,sha256=xxyYV5jPsAptmI0awPO_VGWMaE-y80XMDCB6u87IPaY,6875
 unit_tests/sources/streams/test_streams_core.py,sha256=YOC7XqWFJ13Z4YuO9Nh4AR4AwpJ-s111vqPplFfpxk4,5059
 unit_tests/sources/streams/concurrent/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-unit_tests/sources/streams/concurrent/test_adapters.py,sha256=v8_tv2GCUk73DyoiPEPFDBrTRCsXysR-Cw-DXkQnPn4,14743
-unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py,sha256=6ai_6AeRuiUFB0p5TcFGHMG2eiGFbGrmXhI41Oe5XYQ,1321
-unit_tests/sources/streams/concurrent/test_cursor.py,sha256=sqH8xE3GamETSSVqsdKafziAAm-y_j_MegES_C5ExMM,5790
-unit_tests/sources/streams/concurrent/test_partition_reader.py,sha256=eM5dzfmLKm9Lj-BfQUjAZRhCZzfvhk7AkKpcHGcoPfg,931
+unit_tests/sources/streams/concurrent/test_adapters.py,sha256=Y_c1vKCtGKEzrUSncmpgp0lgFnArmBrIrmLFaOIAxRg,15439
+unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py,sha256=v8yf19_sDgVcWop6WKotahlQiO6B8MwxhGi3AL4vHm8,1375
+unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py,sha256=1578s0CtFVQALrv2slo12RIgNmNwwobJAFWfBre8jdc,23822
+unit_tests/sources/streams/concurrent/test_cursor.py,sha256=xsQ0zHKzU-iRnpTiAMvGRPbiL50zRJerDoloenkhcj0,5818
+unit_tests/sources/streams/concurrent/test_default_stream.py,sha256=VLF46ESoRqcoALYCdrdZ2NDl5s2T1fRRWsYAy2-IwYw,6502
+unit_tests/sources/streams/concurrent/test_partition_reader.py,sha256=2uj7uV3ie0BMb--aa3MUru-f4jLiYUR-Nl0r3EhwxLQ,951
 unit_tests/sources/streams/concurrent/test_state_converter.py,sha256=rvg8becWR1iPdm5TAanZssKj5_iw8dInE_uqmjqghZE,8349
-unit_tests/sources/streams/concurrent/test_thread_based_concurrent_stream.py,sha256=_jBMJIZ6Hu9mWX4v9SRUdtxvgntA-rQpNbbygBi6HXA,11629
+unit_tests/sources/streams/concurrent/test_thread_pool_manager.py,sha256=7L9Sv7VXULOHx3-KSFwFtzAY1X96wcPiPKGq38BQEVg,3699
 unit_tests/sources/streams/concurrent/scenarios/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py,sha256=x77AQf8_O4dQ2aF1o800CzI0hOEyU8ayxoNdSOvxkhM,10495
-unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=FdgEO-bWA_IDFIJb0W83qE4QCCZ8eexbn_Mq8LJq0iE,5040
-unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py,sha256=svWqPyIAJCr-TPm0zFk9_gXB8hluVQuet6TqbV6yW3g,14096
+unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=OD_9R5fHt5Nf7hH8m28-UDoZJkY8iUBJLI73kd-u2BE,5794
+unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py,sha256=v0yP5MRGYJAb9bp2yXnp5yUmYKJ6aAKjHcNHigL_ONY,13981
 unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py,sha256=sQpvIJa5-Iv03KZfC2sP2zB8XSPCZAjLpUMpNBOA-xM,3897
-unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py,sha256=kiZ6VvQywg073FtrpP9AD3yIwSKbalVyfOM2Ksu3sZI,13554
-unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py,sha256=G4Em5zfAd9ExzDaD-86nabxWHj9wn5HT3Mfz37UNiME,5310
+unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py,sha256=KqCLsXB_9rV4hNdSPrNynK3G-UIsipqsZT6X0Z-iM5E,13175
+unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py,sha256=aMtEOpCkxH-v2BBOYj4xABzPKcDYh_jieGfaIp4hy9w,5727
 unit_tests/sources/streams/concurrent/scenarios/utils.py,sha256=Pl1F4asW8AvV6bV5W3Qg21GiLqfdMT_rOt1CsFA0aVM,1953
 unit_tests/sources/streams/http/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/streams/http/test_availability_strategy.py,sha256=kuQJ5FIc4lffpHmEUVzvoN1QXQzvz8WEkFvzHItiipg,6063
@@ -411,8 +423,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
 unit_tests/utils/test_stream_status_utils.py,sha256=Xr8MZ2HWgTVIyMbywDvuYkRaUF4RZLQOT8-JjvcfR24,2970
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.54.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.54.0.dist-info/METADATA,sha256=nq675Flnxy5fGE4WB4NNbRqlRvnXUWrEBEk0F4JYpn8,11983
-airbyte_cdk-0.54.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-airbyte_cdk-0.54.0.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.54.0.dist-info/RECORD,,
+airbyte_cdk-0.55.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.55.0.dist-info/METADATA,sha256=2r6pncfFhwFGRcafz2bnvxd-3BwRcgkvHktsJhG0Z8Q,11983
+airbyte_cdk-0.55.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+airbyte_cdk-0.55.0.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.55.0.dist-info/RECORD,,

unit_tests/sources/concurrent_source/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#

unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py

@@ -0,0 +1,105 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+import logging
+from typing import Any, List, Mapping, Optional, Tuple
+from unittest.mock import Mock
+
+import freezegun
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    AirbyteStream,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    DestinationSyncMode,
+    SyncMode,
+)
+from airbyte_cdk.models import Type as MessageType
+from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
+from airbyte_cdk.sources.message import InMemoryMessageRepository
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade
+from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor
+
+
+class _MockSource(ConcurrentSourceAdapter):
+    def __init__(self, concurrent_source, _streams_to_is_concurrent, logger):
+        super().__init__(concurrent_source)
+        self._streams_to_is_concurrent = _streams_to_is_concurrent
+        self._logger = logger
+
+    message_repository = InMemoryMessageRepository()
+
+    def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
+        raise NotImplementedError
+
+    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
+        return [
+            StreamFacade.create_from_stream(s, self, self._logger, None, NoopCursor()) if is_concurrent else s
+            for s, is_concurrent in self._streams_to_is_concurrent.items()
+        ]
+
+
+@freezegun.freeze_time("2020-01-01T00:00:00")
+def test_concurrent_source_adapter():
+    concurrent_source = Mock()
+    message_from_concurrent_stream = AirbyteMessage(
+        type=MessageType.RECORD,
+        record=AirbyteRecordMessage(
+            stream="s2",
+            data={"data": 2},
+            emitted_at=1577836800000,
+        ),
+    )
+    concurrent_source.read.return_value = iter([message_from_concurrent_stream])
+    regular_stream = _mock_stream("s1", [{"data": 1}])
+    concurrent_stream = _mock_stream("s2", [])
+    unavailable_stream = _mock_stream("s3", [{"data": 3}], False)
+    concurrent_stream.name = "s2"
+    logger = Mock()
+    adapter = _MockSource(concurrent_source, {regular_stream: False, concurrent_stream: True, unavailable_stream: False}, logger)
+
+    messages = list(adapter.read(logger, {}, _configured_catalog([regular_stream, concurrent_stream, unavailable_stream])))
+    records = [m for m in messages if m.type == MessageType.RECORD]
+
+    expected_records = [
+        message_from_concurrent_stream,
+        AirbyteMessage(
+            type=MessageType.RECORD,
+            record=AirbyteRecordMessage(
+                stream="s1",
+                data={"data": 1},
+                emitted_at=1577836800000,
+            ),
+        ),
+    ]
+
+    assert records == expected_records
+
+
+def _mock_stream(name: str, data=[], available: bool = True):
+    s = Mock()
+    s.name = name
+    s.as_airbyte_stream.return_value = AirbyteStream(
+        name=name,
+        json_schema={},
+        supported_sync_modes=[SyncMode.full_refresh],
+    )
+    s.check_availability.return_value = (True, None) if available else (False, "not available")
+    s.read_full_refresh.return_value = iter(data)
+    s.primary_key = None
+    return s
+
+
+def _configured_catalog(streams: List[Stream]):
+    return ConfiguredAirbyteCatalog(
+        streams=[
+            ConfiguredAirbyteStream(
+                stream=stream.as_airbyte_stream(),
+                sync_mode=SyncMode.full_refresh,
+                destination_sync_mode=DestinationSyncMode.overwrite,
+            )
+            for stream in streams
+        ]
+    )

unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py

@@ -1,11 +1,14 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+import concurrent
 import logging
 from typing import Any, List, Mapping, Optional, Tuple, Union
 
 from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, DestinationSyncMode, SyncMode
-from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource
+from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
+from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.streams import Stream
@@ -14,6 +17,7 @@ from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, Curs
 from airbyte_cdk.sources.streams.concurrent.state_converter import EpochValueConcurrentStreamStateConverter
 from airbyte_protocol.models import ConfiguredAirbyteStream
 from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder
+from unit_tests.sources.streams.concurrent.scenarios.thread_based_concurrent_stream_source_builder import NeverLogSliceLogger
 
 _NO_STATE = None
 
@@ -22,18 +26,21 @@ class StreamFacadeConcurrentConnectorStateConverter(EpochValueConcurrentStreamSt
     pass
 
 
-class StreamFacadeSource(AbstractSource):
+class StreamFacadeSource(ConcurrentSourceAdapter):
     def __init__(
         self,
         streams: List[Stream],
-        max_workers: int,
+        threadpool: concurrent.futures.ThreadPoolExecutor,
         cursor_field: Optional[CursorField] = None,
         cursor_boundaries: Optional[Tuple[str, str]] = None,
         input_state: Optional[List[Mapping[str, Any]]] = _NO_STATE,
     ):
-        self._streams = streams
-        self._max_workers = max_workers
         self._message_repository = InMemoryMessageRepository()
+        threadpool_manager = ThreadPoolManager(threadpool, streams[0].logger)
+        concurrent_source = ConcurrentSource(threadpool_manager, streams[0].logger, NeverLogSliceLogger(), self._message_repository)
+        super().__init__(concurrent_source)
+        self._streams = streams
+        self._threadpool = threadpool_manager
         self._cursor_field = cursor_field
         self._cursor_boundaries = cursor_boundaries
         self._state = [AirbyteStateMessage.parse_obj(s) for s in input_state] if input_state else None
@@ -49,7 +56,6 @@ class StreamFacadeSource(AbstractSource):
                 stream,
                 self,
                 stream.logger,
-                self._max_workers,
                 state_converter.get_concurrent_stream_state(state_manager.get_stream_state(stream.name, stream.namespace)),
                 ConcurrentCursor(
                     stream.name,
@@ -115,4 +121,5 @@ class StreamFacadeSourceBuilder(SourceBuilder[StreamFacadeSource]):
         return self
 
     def build(self, configured_catalog: Optional[Mapping[str, Any]]) -> StreamFacadeSource:
-        return StreamFacadeSource(self._streams, self._max_workers, self._cursor_field, self._cursor_boundaries, self._input_state)
+        threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=self._max_workers, thread_name_prefix="workerpool")
+        return StreamFacadeSource(self._streams, threadpool, self._cursor_field, self._cursor_boundaries, self._input_state)