airbyte-cdk 0.59.1__tar.gz → 0.60.0__tar.gz

{airbyte-cdk-0.59.1 → airbyte-cdk-0.60.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.59.1
+Version: 0.60.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.59.1 → airbyte-cdk-0.60.0}/airbyte_cdk/entrypoint.py

@@ -212,7 +212,9 @@ def launch(source: Source, args: List[str]) -> None:
     source_entrypoint = AirbyteEntrypoint(source)
     parsed_args = source_entrypoint.parse_args(args)
     for message in source_entrypoint.run(parsed_args):
-        print(message)
+        # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and
+        # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time
+        print(f"{message}\n", end="")
 
 
 def _init_internal_request_filter() -> None:

{airbyte-cdk-0.59.1 → airbyte-cdk-0.60.0}/airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py

@@ -10,7 +10,7 @@ from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
-from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade
+from airbyte_cdk.sources.streams.concurrent.abstract_stream_facade import AbstractStreamFacade
 
 
 class ConcurrentSourceAdapter(AbstractSource, ABC):
@@ -58,6 +58,6 @@ class ConcurrentSourceAdapter(AbstractSource, ABC):
                     f"The stream {configured_stream.stream.name} no longer exists in the configuration. "
                     f"Refresh the schema in replication settings and remove this stream from future sync attempts."
                 )
-            if isinstance(stream_instance, StreamFacade):
-                abstract_streams.append(stream_instance._abstract_stream)
+            if isinstance(stream_instance, AbstractStreamFacade):
+                abstract_streams.append(stream_instance.get_underlying_stream())
         return abstract_streams

airbyte-cdk-0.60.0/airbyte_cdk/sources/file_based/availability_strategy/__init__.py

@@ -0,0 +1,7 @@
+from .abstract_file_based_availability_strategy import (
+    AbstractFileBasedAvailabilityStrategy,
+    AbstractFileBasedAvailabilityStrategyWrapper,
+)
+from .default_file_based_availability_strategy import DefaultFileBasedAvailabilityStrategy
+
+__all__ = ["AbstractFileBasedAvailabilityStrategy", "AbstractFileBasedAvailabilityStrategyWrapper", "DefaultFileBasedAvailabilityStrategy"]

{airbyte-cdk-0.59.1 → airbyte-cdk-0.60.0}/airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py

@@ -8,6 +8,12 @@ from typing import TYPE_CHECKING, Optional, Tuple
 
 from airbyte_cdk.sources import Source
 from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
+from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
+    AbstractAvailabilityStrategy,
+    StreamAvailability,
+    StreamAvailable,
+    StreamUnavailable,
+)
 from airbyte_cdk.sources.streams.core import Stream
 
 if TYPE_CHECKING:
@@ -35,3 +41,17 @@ class AbstractFileBasedAvailabilityStrategy(AvailabilityStrategy):
         Returns (True, None) if successful, otherwise (False, <error message>).
         """
         ...
+
+
+class AbstractFileBasedAvailabilityStrategyWrapper(AbstractAvailabilityStrategy):
+    def __init__(self, stream: "AbstractFileBasedStream"):
+        self.stream = stream
+
+    def check_availability(self, logger: logging.Logger) -> StreamAvailability:
+        is_available, reason = self.stream.availability_strategy.check_availability(self.stream, logger, None)
+        if is_available:
+            return StreamAvailable()
+        return StreamUnavailable(reason or "")
+
+    def check_availability_and_parsability(self, logger: logging.Logger) -> Tuple[bool, Optional[str]]:
+        return self.stream.availability_strategy.check_availability_and_parsability(self.stream, logger, None)

{airbyte-cdk-0.59.1 → airbyte-cdk-0.60.0}/airbyte_cdk/sources/file_based/file_based_source.py

@@ -8,8 +8,18 @@ from abc import ABC
 from collections import Counter
 from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Type, Union
 
-from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification
-from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.logger import AirbyteLogFormatter, init_logger
+from airbyte_cdk.models import (
+    AirbyteMessage,
+    AirbyteStateMessage,
+    ConfiguredAirbyteCatalog,
+    ConnectorSpecification,
+    FailureType,
+    Level,
+    SyncMode,
+)
+from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource
+from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
 from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy, DefaultFileBasedAvailabilityStrategy
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ValidationPolicy
@@ -20,19 +30,33 @@ from airbyte_cdk.sources.file_based.file_types import default_parsers
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.schema_validation_policies import DEFAULT_SCHEMA_VALIDATION_POLICIES, AbstractSchemaValidationPolicy
 from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream, DefaultFileBasedStream
+from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamFacade
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedNoopCursor
 from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
 from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor
+from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
+from airbyte_cdk.sources.source import TState
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.utils.analytics_message import create_analytics_message
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 from pydantic.error_wrappers import ValidationError
 
+DEFAULT_CONCURRENCY = 100
+MAX_CONCURRENCY = 100
+INITIAL_N_PARTITIONS = MAX_CONCURRENCY // 2
+
+
+class FileBasedSource(ConcurrentSourceAdapter, ABC):
+    # We make each source override the concurrency level to give control over when they are upgraded.
+    _concurrency_level = None
 
-class FileBasedSource(AbstractSource, ABC):
     def __init__(
         self,
         stream_reader: AbstractFileBasedStreamReader,
         spec_class: Type[AbstractFileBasedSpec],
-        catalog_path: Optional[str] = None,
+        catalog: Optional[ConfiguredAirbyteCatalog],
+        config: Optional[Mapping[str, Any]],
+        state: Optional[TState],
         availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy] = None,
         discovery_policy: AbstractDiscoveryPolicy = DefaultDiscoveryPolicy(),
         parsers: Mapping[Type[Any], FileTypeParser] = default_parsers,
@@ -41,15 +65,29 @@ class FileBasedSource(AbstractSource, ABC):
     ):
         self.stream_reader = stream_reader
         self.spec_class = spec_class
+        self.config = config
+        self.catalog = catalog
+        self.state = state
         self.availability_strategy = availability_strategy or DefaultFileBasedAvailabilityStrategy(stream_reader)
         self.discovery_policy = discovery_policy
         self.parsers = parsers
         self.validation_policies = validation_policies
-        catalog = self.read_catalog(catalog_path) if catalog_path else None
         self.stream_schemas = {s.stream.name: s.stream.json_schema for s in catalog.streams} if catalog else {}
         self.cursor_cls = cursor_cls
-        self.logger = logging.getLogger(f"airbyte.{self.name}")
+        self.logger = init_logger(f"airbyte.{self.name}")
         self.errors_collector: FileBasedErrorsCollector = FileBasedErrorsCollector()
+        self._message_repository: Optional[MessageRepository] = None
+        concurrent_source = ConcurrentSource.create(
+            MAX_CONCURRENCY, INITIAL_N_PARTITIONS, self.logger, self._slice_logger, self.message_repository
+        )
+        self._state = None
+        super().__init__(concurrent_source)
+
+    @property
+    def message_repository(self) -> MessageRepository:
+        if self._message_repository is None:
+            self._message_repository = InMemoryMessageRepository(Level(AirbyteLogFormatter.level_mapping[self.logger.level]))
+        return self._message_repository
 
     def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
         """
@@ -61,7 +99,15 @@ class FileBasedSource(AbstractSource, ABC):
 
         Otherwise, the "error" object should describe what went wrong.
         """
-        streams = self.streams(config)
+        try:
+            streams = self.streams(config)
+        except Exception as config_exception:
+            raise AirbyteTracedException(
+                internal_message="Please check the logged errors for more information.",
+                message=FileBasedSourceError.CONFIG_VALIDATION_ERROR.value,
+                exception=AirbyteTracedException(exception=config_exception),
+                failure_type=FailureType.config_error,
+            )
         if len(streams) == 0:
             return (
                 False,
@@ -80,7 +126,7 @@ class FileBasedSource(AbstractSource, ABC):
                     reason,
                 ) = stream.availability_strategy.check_availability_and_parsability(stream, logger, self)
             except Exception:
-                errors.append(f"Unable to connect to stream {stream} - {''.join(traceback.format_exc())}")
+                errors.append(f"Unable to connect to stream {stream.name} - {''.join(traceback.format_exc())}")
             else:
                 if not stream_is_available and reason:
                     errors.append(reason)
@@ -91,10 +137,26 @@ class FileBasedSource(AbstractSource, ABC):
         """
         Return a list of this source's streams.
         """
+        file_based_streams = self._get_file_based_streams(config)
+
+        configured_streams: List[Stream] = []
+
+        for stream in file_based_streams:
+            sync_mode = self._get_sync_mode_from_catalog(stream)
+            if sync_mode == SyncMode.full_refresh and hasattr(self, "_concurrency_level") and self._concurrency_level is not None:
+                configured_streams.append(
+                    FileBasedStreamFacade.create_from_stream(stream, self, self.logger, None, FileBasedNoopCursor(stream.config))
+                )
+            else:
+                configured_streams.append(stream)
+
+        return configured_streams
+
+    def _get_file_based_streams(self, config: Mapping[str, Any]) -> List[AbstractFileBasedStream]:
         try:
             parsed_config = self._get_parsed_config(config)
             self.stream_reader.config = parsed_config
-            streams: List[Stream] = []
+            streams: List[AbstractFileBasedStream] = []
             for stream_config in parsed_config.streams:
                 self._validate_input_schema(stream_config)
                 streams.append(
@@ -115,6 +177,14 @@ class FileBasedSource(AbstractSource, ABC):
         except ValidationError as exc:
             raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR) from exc
 
+    def _get_sync_mode_from_catalog(self, stream: Stream) -> Optional[SyncMode]:
+        if self.catalog:
+            for catalog_stream in self.catalog.streams:
+                if stream.name == catalog_stream.stream.name:
+                    return catalog_stream.sync_mode
+            raise RuntimeError(f"No sync mode was found for {stream.name}.")
+        return None
+
     def read(
         self,
         logger: logging.Logger,

{airbyte-cdk-0.59.1 → airbyte-cdk-0.60.0}/airbyte_cdk/sources/file_based/file_types/csv_parser.py

@@ -10,6 +10,7 @@ from collections import defaultdict
 from functools import partial
 from io import IOBase
 from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Set, Tuple
+from uuid import uuid4
 
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat, CsvHeaderAutogenerated, CsvHeaderUserProvided, InferenceType
@@ -38,8 +39,10 @@ class _CsvReader:
 
         # Formats are configured individually per-stream so a unique dialect should be registered for each stream.
         # We don't unregister the dialect because we are lazily parsing each csv file to generate records
-        # This will potentially be a problem if we ever process multiple streams concurrently
-        dialect_name = config.name + DIALECT_NAME
+        # Give each stream's dialect a unique name; otherwise, when we are doing a concurrent sync we can end up
+        # with a race condition where a thread attempts to use a dialect before a separate thread has finished
+        # registering it.
+        dialect_name = f"{config.name}_{str(uuid4())}_{DIALECT_NAME}"
         csv.register_dialect(
             dialect_name,
             delimiter=config_format.delimiter,

airbyte-cdk-0.60.0/airbyte_cdk/sources/file_based/stream/concurrent/adapters.py

@@ -0,0 +1,322 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import copy
+import logging
+from functools import lru_cache
+from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
+
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode, Type
+from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
+from airbyte_cdk.sources.file_based.availability_strategy import (
+    AbstractFileBasedAvailabilityStrategy,
+    AbstractFileBasedAvailabilityStrategyWrapper,
+)
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
+from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedNoopCursor
+from airbyte_cdk.sources.file_based.types import StreamSlice
+from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.streams.concurrent.abstract_stream_facade import AbstractStreamFacade
+from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
+from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage
+from airbyte_cdk.sources.streams.concurrent.helpers import get_cursor_field_from_stream, get_primary_key_from_stream
+from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
+from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
+from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
+from airbyte_cdk.sources.streams.core import StreamData
+from airbyte_cdk.sources.utils.schema_helpers import InternalConfig
+from airbyte_cdk.sources.utils.slice_logger import SliceLogger
+from deprecated.classic import deprecated
+
+"""
+This module contains adapters to help enabling concurrency on File-based Stream objects without needing to migrate to AbstractStream
+"""
+
+
+@deprecated("This class is experimental. Use at your own risk.")
+class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBasedStream):
+    @classmethod
+    def create_from_stream(
+        cls,
+        stream: AbstractFileBasedStream,
+        source: AbstractSource,
+        logger: logging.Logger,
+        state: Optional[MutableMapping[str, Any]],
+        cursor: FileBasedNoopCursor,
+    ) -> "FileBasedStreamFacade":
+        """
+        Create a ConcurrentStream from a FileBasedStream object.
+        """
+        pk = get_primary_key_from_stream(stream.primary_key)
+        cursor_field = get_cursor_field_from_stream(stream)
+
+        if not source.message_repository:
+            raise ValueError(
+                "A message repository is required to emit non-record messages. Please set the message repository on the source."
+            )
+
+        message_repository = source.message_repository
+        return FileBasedStreamFacade(
+            DefaultStream(  # type: ignore
+                partition_generator=FileBasedStreamPartitionGenerator(
+                    stream,
+                    message_repository,
+                    SyncMode.full_refresh if isinstance(cursor, FileBasedNoopCursor) else SyncMode.incremental,
+                    [cursor_field] if cursor_field is not None else None,
+                    state,
+                    cursor,
+                ),
+                name=stream.name,
+                json_schema=stream.get_json_schema(),
+                availability_strategy=AbstractFileBasedAvailabilityStrategyWrapper(stream),
+                primary_key=pk,
+                cursor_field=cursor_field,
+                logger=logger,
+                namespace=stream.namespace,
+            ),
+            stream,
+            cursor,
+            logger=logger,
+            slice_logger=source._slice_logger,
+        )
+
+    def __init__(
+        self,
+        stream: DefaultStream,
+        legacy_stream: AbstractFileBasedStream,
+        cursor: FileBasedNoopCursor,
+        slice_logger: SliceLogger,
+        logger: logging.Logger,
+    ):
+        """
+        :param stream: The underlying AbstractStream
+        """
+        # super().__init__(stream, legacy_stream, cursor, slice_logger, logger)
+        self._abstract_stream = stream
+        self._legacy_stream = legacy_stream
+        self._cursor = cursor
+        self._slice_logger = slice_logger
+        self._logger = logger
+        self.catalog_schema = legacy_stream.catalog_schema
+        self.config = legacy_stream.config
+        self.validation_policy = legacy_stream.validation_policy
+
+    @property
+    def cursor_field(self) -> Union[str, List[str]]:
+        if self._abstract_stream.cursor_field is None:
+            return []
+        else:
+            return self._abstract_stream.cursor_field
+
+    @property
+    def name(self) -> str:
+        return self._abstract_stream.name
+
+    @property
+    def supports_incremental(self) -> bool:
+        return self._legacy_stream.supports_incremental
+
+    @property
+    def availability_strategy(self) -> AbstractFileBasedAvailabilityStrategy:
+        return self._legacy_stream.availability_strategy
+
+    @lru_cache(maxsize=None)
+    def get_json_schema(self) -> Mapping[str, Any]:
+        return self._abstract_stream.get_json_schema()
+
+    @property
+    def primary_key(self) -> PrimaryKeyType:
+        return self._legacy_stream.config.primary_key or self.get_parser().get_parser_defined_primary_key(self._legacy_stream.config)
+
+    def get_parser(self) -> FileTypeParser:
+        return self._legacy_stream.get_parser()
+
+    def get_files(self) -> Iterable[RemoteFile]:
+        return self._legacy_stream.get_files()
+
+    def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[Mapping[str, Any]]:
+        yield from self._legacy_stream.read_records_from_slice(stream_slice)
+
+    def compute_slices(self) -> Iterable[Optional[StreamSlice]]:
+        return self._legacy_stream.compute_slices()
+
+    def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]:
+        return self._legacy_stream.infer_schema(files)
+
+    def get_underlying_stream(self) -> DefaultStream:
+        return self._abstract_stream
+
+    def read_full_refresh(
+        self,
+        cursor_field: Optional[List[str]],
+        logger: logging.Logger,
+        slice_logger: SliceLogger,
+    ) -> Iterable[StreamData]:
+        """
+        Read full refresh. Delegate to the underlying AbstractStream, ignoring all the parameters
+        :param cursor_field: (ignored)
+        :param logger: (ignored)
+        :param slice_logger: (ignored)
+        :return: Iterable of StreamData
+        """
+        yield from self._read_records()
+
+    def read_incremental(
+        self,
+        cursor_field: Optional[List[str]],
+        logger: logging.Logger,
+        slice_logger: SliceLogger,
+        stream_state: MutableMapping[str, Any],
+        state_manager: ConnectorStateManager,
+        per_stream_state_enabled: bool,
+        internal_config: InternalConfig,
+    ) -> Iterable[StreamData]:
+        yield from self._read_records()
+
+    def read_records(
+        self,
+        sync_mode: SyncMode,
+        cursor_field: Optional[List[str]] = None,
+        stream_slice: Optional[Mapping[str, Any]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
+    ) -> Iterable[StreamData]:
+        try:
+            yield from self._read_records()
+        except Exception as exc:
+            if hasattr(self._cursor, "state"):
+                state = str(self._cursor.state)
+            else:
+                # This shouldn't happen if the ConcurrentCursor was used
+                state = "unknown; no state attribute was available on the cursor"
+            yield AirbyteMessage(
+                type=Type.LOG, log=AirbyteLogMessage(level=Level.ERROR, message=f"Cursor State at time of exception: {state}")
+            )
+            raise exc
+
+    def _read_records(self) -> Iterable[StreamData]:
+        for partition in self._abstract_stream.generate_partitions():
+            if self._slice_logger.should_log_slice_message(self._logger):
+                yield self._slice_logger.create_slice_log_message(partition.to_slice())
+            for record in partition.read():
+                yield record.data
+
+
+class FileBasedStreamPartition(Partition):
+    def __init__(
+        self,
+        stream: AbstractFileBasedStream,
+        _slice: Optional[Mapping[str, Any]],
+        message_repository: MessageRepository,
+        sync_mode: SyncMode,
+        cursor_field: Optional[List[str]],
+        state: Optional[MutableMapping[str, Any]],
+        cursor: FileBasedNoopCursor,
+    ):
+        self._stream = stream
+        self._slice = _slice
+        self._message_repository = message_repository
+        self._sync_mode = sync_mode
+        self._cursor_field = cursor_field
+        self._state = state
+        self._cursor = cursor
+        self._is_closed = False
+
+    def read(self) -> Iterable[Record]:
+        try:
+            for record_data in self._stream.read_records(
+                cursor_field=self._cursor_field,
+                sync_mode=SyncMode.full_refresh,
+                stream_slice=copy.deepcopy(self._slice),
+                stream_state=self._state,
+            ):
+                if isinstance(record_data, Mapping):
+                    data_to_return = dict(record_data)
+                    self._stream.transformer.transform(data_to_return, self._stream.get_json_schema())
+                    yield Record(data_to_return, self.stream_name())
+                else:
+                    self._message_repository.emit_message(record_data)
+        except Exception as e:
+            display_message = self._stream.get_error_display_message(e)
+            if display_message:
+                raise ExceptionWithDisplayMessage(display_message) from e
+            else:
+                raise e
+
+    def to_slice(self) -> Optional[Mapping[str, Any]]:
+        if self._slice is None:
+            return None
+        assert (
+            len(self._slice["files"]) == 1
+        ), f"Expected 1 file per partition but got {len(self._slice['files'])} for stream {self.stream_name()}"
+        file = self._slice["files"][0]
+        return {"files": [file]}
+
+    def close(self) -> None:
+        self._cursor.close_partition(self)
+        self._is_closed = True
+
+    def is_closed(self) -> bool:
+        return self._is_closed
+
+    def __hash__(self) -> int:
+        if self._slice:
+            # Convert the slice to a string so that it can be hashed
+            if len(self._slice["files"]) != 1:
+                raise ValueError(
+                    f"Slices for file-based streams should be of length 1, but got {len(self._slice['files'])}. This is unexpected. Please contact Support."
+                )
+            else:
+                s = f"{self._slice['files'][0].last_modified.strftime('%Y-%m-%dT%H:%M:%S.%fZ')}_{self._slice['files'][0].uri}"
+            return hash((self._stream.name, s))
+        else:
+            return hash(self._stream.name)
+
+    def stream_name(self) -> str:
+        return self._stream.name
+
+    def __repr__(self) -> str:
+        return f"FileBasedStreamPartition({self._stream.name}, {self._slice})"
+
+
+class FileBasedStreamPartitionGenerator(PartitionGenerator):
+    def __init__(
+        self,
+        stream: AbstractFileBasedStream,
+        message_repository: MessageRepository,
+        sync_mode: SyncMode,
+        cursor_field: Optional[List[str]],
+        state: Optional[MutableMapping[str, Any]],
+        cursor: FileBasedNoopCursor,
+    ):
+        self._stream = stream
+        self._message_repository = message_repository
+        self._sync_mode = sync_mode
+        self._cursor_field = cursor_field
+        self._state = state
+        self._cursor = cursor
+
+    def generate(self) -> Iterable[FileBasedStreamPartition]:
+        pending_partitions = []
+        for _slice in self._stream.stream_slices(sync_mode=self._sync_mode, cursor_field=self._cursor_field, stream_state=self._state):
+            if _slice is not None:
+                pending_partitions.extend(
+                    [
+                        FileBasedStreamPartition(
+                            self._stream,
+                            {"files": [copy.deepcopy(f)]},
+                            self._message_repository,
+                            self._sync_mode,
+                            self._cursor_field,
+                            self._state,
+                            self._cursor,
+                        )
+                        for f in _slice.get("files", [])
+                    ]
+                )
+        self._cursor.set_pending_partitions(pending_partitions)
+        yield from pending_partitions

airbyte-cdk-0.60.0/airbyte_cdk/sources/file_based/stream/concurrent/cursor.py

@@ -0,0 +1,87 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+import logging
+from abc import abstractmethod
+from datetime import datetime
+from typing import Any, Iterable, MutableMapping
+
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
+from airbyte_cdk.sources.file_based.types import StreamState
+from airbyte_cdk.sources.streams.concurrent.cursor import Cursor
+from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
+from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
+
+
+class AbstractFileBasedConcurrentCursor(Cursor, AbstractFileBasedCursor):
+    @property
+    @abstractmethod
+    def state(self) -> MutableMapping[str, Any]:
+        ...
+
+    @abstractmethod
+    def add_file(self, file: RemoteFile) -> None:
+        ...
+
+    @abstractmethod
+    def set_initial_state(self, value: StreamState) -> None:
+        ...
+
+    @abstractmethod
+    def get_state(self) -> MutableMapping[str, Any]:
+        ...
+
+    @abstractmethod
+    def get_start_time(self) -> datetime:
+        ...
+
+    @abstractmethod
+    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+        ...
+
+    @abstractmethod
+    def observe(self, record: Record) -> None:
+        ...
+
+    @abstractmethod
+    def close_partition(self, partition: Partition) -> None:
+        ...
+
+    @abstractmethod
+    def set_pending_partitions(self, partitions: Iterable[Partition]) -> None:
+        ...
+
+
+class FileBasedNoopCursor(AbstractFileBasedConcurrentCursor):
+    def __init__(self, stream_config: FileBasedStreamConfig, **kwargs: Any):
+        pass
+
+    @property
+    def state(self) -> MutableMapping[str, Any]:
+        return {}
+
+    def add_file(self, file: RemoteFile) -> None:
+        return None
+
+    def set_initial_state(self, value: StreamState) -> None:
+        return None
+
+    def get_state(self) -> MutableMapping[str, Any]:
+        return {}
+
+    def get_start_time(self) -> datetime:
+        return datetime.min
+
+    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+        return []
+
+    def observe(self, record: Record) -> None:
+        return None
+
+    def close_partition(self, partition: Partition) -> None:
+        return None
+
+    def set_pending_partitions(self, partitions: Iterable[Partition]) -> None:
+        return None