airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev1__py3-none-any.whl

airbyte_cdk/sources/file_based/schema_helpers.py

@@ -8,12 +8,20 @@ from enum import Enum
 from functools import total_ordering
 from typing import Any, Dict, List, Literal, Mapping, Optional, Tuple, Type, Union
 
-from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError, SchemaInferenceError
+from airbyte_cdk.sources.file_based.exceptions import (
+    ConfigValidationError,
+    FileBasedSourceError,
+    SchemaInferenceError,
+)
 
 JsonSchemaSupportedType = Union[List[str], Literal["string"], str]
 SchemaType = Mapping[str, Mapping[str, JsonSchemaSupportedType]]
 
 schemaless_schema = {"type": "object", "properties": {"data": {"type": "object"}}}
+file_transfer_schema = {
+    "type": "object",
+    "properties": {"data": {"type": "object"}, "file": {"type": "object"}},
+}
 
 
 @total_ordering
@@ -117,14 +125,23 @@ def _is_valid_type(t: JsonSchemaSupportedType) -> bool:
 
 
 def _choose_wider_type(key: str, t1: Mapping[str, Any], t2: Mapping[str, Any]) -> Mapping[str, Any]:
-    if (t1["type"] == "array" or t2["type"] == "array") and t1 != t2:
+    t1_type = t1["type"]
+    t2_type = t2["type"]
+
+    if (t1_type == "array" or t2_type == "array") and t1 != t2:
         raise SchemaInferenceError(
             FileBasedSourceError.SCHEMA_INFERENCE_ERROR,
             details="Cannot merge schema for unequal array types.",
             key=key,
             detected_types=f"{t1},{t2}",
         )
-    elif (t1["type"] == "object" or t2["type"] == "object") and t1 != t2:
+    # Schemas can still be merged if a key contains a null value in either t1 or t2, but it is still an object
+    elif (
+        (t1_type == "object" or t2_type == "object")
+        and t1_type != "null"
+        and t2_type != "null"
+        and t1 != t2
+    ):
         raise SchemaInferenceError(
             FileBasedSourceError.SCHEMA_INFERENCE_ERROR,
             details="Cannot merge schema for unequal object types.",
@@ -132,12 +149,19 @@ def _choose_wider_type(key: str, t1: Mapping[str, Any], t2: Mapping[str, Any]) -
             detected_types=f"{t1},{t2}",
         )
     else:
-        comparable_t1 = get_comparable_type(TYPE_PYTHON_MAPPING[t1["type"]][0])  # accessing the type_mapping value
-        comparable_t2 = get_comparable_type(TYPE_PYTHON_MAPPING[t2["type"]][0])  # accessing the type_mapping value
+        comparable_t1 = get_comparable_type(
+            TYPE_PYTHON_MAPPING[t1_type][0]
+        )  # accessing the type_mapping value
+        comparable_t2 = get_comparable_type(
+            TYPE_PYTHON_MAPPING[t2_type][0]
+        )  # accessing the type_mapping value
         if not comparable_t1 and comparable_t2:
-            raise SchemaInferenceError(FileBasedSourceError.UNRECOGNIZED_TYPE, key=key, detected_types=f"{t1},{t2}")
+            raise SchemaInferenceError(
+                FileBasedSourceError.UNRECOGNIZED_TYPE, key=key, detected_types=f"{t1},{t2}"
+            )
         return max(
-            [t1, t2], key=lambda x: ComparableType(get_comparable_type(TYPE_PYTHON_MAPPING[x["type"]][0]))
+            [t1, t2],
+            key=lambda x: ComparableType(get_comparable_type(TYPE_PYTHON_MAPPING[x["type"]][0])),
         )  # accessing the type_mapping value
 
 
@@ -200,7 +224,8 @@ def _parse_json_input(input_schema: Union[str, Mapping[str, str]]) -> Optional[M
             schema = input_schema
         if not all(isinstance(s, str) for s in schema.values()):
             raise ConfigValidationError(
-                FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA, details="Invalid input schema; nested schemas are not supported."
+                FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA,
+                details="Invalid input schema; nested schemas are not supported.",
             )
 
     except json.decoder.JSONDecodeError:
@@ -209,7 +234,9 @@ def _parse_json_input(input_schema: Union[str, Mapping[str, str]]) -> Optional[M
     return schema
 
 
-def type_mapping_to_jsonschema(input_schema: Optional[Union[str, Mapping[str, str]]]) -> Optional[Mapping[str, Any]]:
+def type_mapping_to_jsonschema(
+    input_schema: Optional[Union[str, Mapping[str, str]]],
+) -> Optional[Mapping[str, Any]]:
     """
     Return the user input schema (type mapping), transformed to JSON Schema format.
 
@@ -236,7 +263,8 @@ def type_mapping_to_jsonschema(input_schema: Optional[Union[str, Mapping[str, st
 
         if not _json_schema_type:
             raise ConfigValidationError(
-                FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA, details=f"Invalid type '{type_name}' for property '{col_name}'."
+                FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA,
+                details=f"Invalid type '{type_name}' for property '{col_name}'.",
             )
 
         json_schema_type = _json_schema_type[0]

airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py

@@ -1,4 +1,6 @@
-from airbyte_cdk.sources.file_based.schema_validation_policies.abstract_schema_validation_policy import AbstractSchemaValidationPolicy
+from airbyte_cdk.sources.file_based.schema_validation_policies.abstract_schema_validation_policy import (
+    AbstractSchemaValidationPolicy,
+)
 from airbyte_cdk.sources.file_based.schema_validation_policies.default_schema_validation_policies import (
     DEFAULT_SCHEMA_VALIDATION_POLICIES,
     EmitRecordPolicy,

airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py

@@ -11,7 +11,9 @@ class AbstractSchemaValidationPolicy(ABC):
     validate_schema_before_sync = False  # Whether to verify that records conform to the schema during the stream's availabilty check
 
     @abstractmethod
-    def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool:
+    def record_passes_validation_policy(
+        self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]
+    ) -> bool:
         """
         Return True if the record passes the user's validation policy.
         """

airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py

@@ -5,7 +5,10 @@
 from typing import Any, Mapping, Optional
 
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import ValidationPolicy
-from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, StopSyncPerValidationPolicy
+from airbyte_cdk.sources.file_based.exceptions import (
+    FileBasedSourceError,
+    StopSyncPerValidationPolicy,
+)
 from airbyte_cdk.sources.file_based.schema_helpers import conforms_to_schema
 from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy
 
@@ -13,14 +16,18 @@ from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSc
 class EmitRecordPolicy(AbstractSchemaValidationPolicy):
     name = "emit_record"
 
-    def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool:
+    def record_passes_validation_policy(
+        self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]
+    ) -> bool:
         return True
 
 
 class SkipRecordPolicy(AbstractSchemaValidationPolicy):
     name = "skip_record"
 
-    def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool:
+    def record_passes_validation_policy(
+        self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]
+    ) -> bool:
         return schema is not None and conforms_to_schema(record, schema)
 
 
@@ -28,9 +35,13 @@ class WaitForDiscoverPolicy(AbstractSchemaValidationPolicy):
     name = "wait_for_discover"
     validate_schema_before_sync = True
 
-    def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool:
+    def record_passes_validation_policy(
+        self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]
+    ) -> bool:
         if schema is None or not conforms_to_schema(record, schema):
-            raise StopSyncPerValidationPolicy(FileBasedSourceError.STOP_SYNC_PER_SCHEMA_VALIDATION_POLICY)
+            raise StopSyncPerValidationPolicy(
+                FileBasedSourceError.STOP_SYNC_PER_SCHEMA_VALIDATION_POLICY
+            )
         return True
 
 

airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py

@@ -6,11 +6,24 @@ from abc import abstractmethod
 from functools import cache, cached_property, lru_cache
 from typing import Any, Dict, Iterable, List, Mapping, Optional, Type
 
+from typing_extensions import deprecated
+
+from airbyte_cdk import AirbyteMessage
 from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, PrimaryKeyType
+from airbyte_cdk.sources.file_based.availability_strategy import (
+    AbstractFileBasedAvailabilityStrategy,
+)
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import (
+    FileBasedStreamConfig,
+    PrimaryKeyType,
+)
 from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy
-from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector, FileBasedSourceError, RecordParseError, UndefinedParserError
+from airbyte_cdk.sources.file_based.exceptions import (
+    FileBasedErrorsCollector,
+    FileBasedSourceError,
+    RecordParseError,
+    UndefinedParserError,
+)
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
@@ -18,6 +31,7 @@ from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSc
 from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
 from airbyte_cdk.sources.file_based.types import StreamSlice
 from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.checkpoint import Cursor
 
 
 class AbstractFileBasedStream(Stream):
@@ -32,7 +46,7 @@ class AbstractFileBasedStream(Stream):
       files in the stream.
     - A DiscoveryPolicy that controls the number of concurrent requests sent to the source
       during discover, and the number of files used for schema discovery.
-    - A dictionary of FileType:Parser that holds all of the file types that can be handled
+    - A dictionary of FileType:Parser that holds all the file types that can be handled
       by the stream.
     """
 
@@ -61,8 +75,7 @@ class AbstractFileBasedStream(Stream):
 
     @property
     @abstractmethod
-    def primary_key(self) -> PrimaryKeyType:
-        ...
+    def primary_key(self) -> PrimaryKeyType: ...
 
     @cache
     def list_files(self) -> List[RemoteFile]:
@@ -70,7 +83,7 @@ class AbstractFileBasedStream(Stream):
         List all files that belong to the stream.
 
         The output of this method is cached so we don't need to list the files more than once.
-        This means we won't pick up changes to the files during a sync. This meethod uses the
+        This means we won't pick up changes to the files during a sync. This method uses the
         get_files method which is implemented by the concrete stream class.
         """
         return list(self.get_files())
@@ -88,7 +101,7 @@ class AbstractFileBasedStream(Stream):
         cursor_field: Optional[List[str]] = None,
         stream_slice: Optional[StreamSlice] = None,
         stream_state: Optional[Mapping[str, Any]] = None,
-    ) -> Iterable[Mapping[str, Any]]:
+    ) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
         """
         Yield all records from all remote files in `list_files_for_this_sync`.
         This method acts as an adapter between the generic Stream interface and the file-based's
@@ -99,14 +112,20 @@ class AbstractFileBasedStream(Stream):
         return self.read_records_from_slice(stream_slice)
 
     @abstractmethod
-    def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[Mapping[str, Any]]:
+    def read_records_from_slice(
+        self, stream_slice: StreamSlice
+    ) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
         """
         Yield all records from all remote files in `list_files_for_this_sync`.
         """
         ...
 
     def stream_slices(
-        self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None
+        self,
+        *,
+        sync_mode: SyncMode,
+        cursor_field: Optional[List[str]] = None,
+        stream_state: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[Optional[Mapping[str, Any]]]:
         """
         This method acts as an adapter between the generic Stream interface and the file-based's
@@ -141,20 +160,38 @@ class AbstractFileBasedStream(Stream):
         try:
             return self._parsers[type(self.config.format)]
         except KeyError:
-            raise UndefinedParserError(FileBasedSourceError.UNDEFINED_PARSER, stream=self.name, format=type(self.config.format))
+            raise UndefinedParserError(
+                FileBasedSourceError.UNDEFINED_PARSER,
+                stream=self.name,
+                format=type(self.config.format),
+            )
 
     def record_passes_validation_policy(self, record: Mapping[str, Any]) -> bool:
         if self.validation_policy:
-            return self.validation_policy.record_passes_validation_policy(record=record, schema=self.catalog_schema)
+            return self.validation_policy.record_passes_validation_policy(
+                record=record, schema=self.catalog_schema
+            )
         else:
             raise RecordParseError(
-                FileBasedSourceError.UNDEFINED_VALIDATION_POLICY, stream=self.name, validation_policy=self.config.validation_policy
+                FileBasedSourceError.UNDEFINED_VALIDATION_POLICY,
+                stream=self.name,
+                validation_policy=self.config.validation_policy,
             )
 
     @cached_property
+    @deprecated("Deprecated as of CDK version 3.7.0.")
     def availability_strategy(self) -> AbstractFileBasedAvailabilityStrategy:
         return self._availability_strategy
 
     @property
     def name(self) -> str:
         return self.config.name
+
+    def get_cursor(self) -> Optional[Cursor]:
+        """
+        This is a temporary hack. Because file-based, declarative, and concurrent have _slightly_ different cursor implementations
+        the file-based cursor isn't compatible with the cursor-based iteration flow in core.py top-level CDK. By setting this to
+        None, we defer to the regular incremental checkpoint flow. Once all cursors are consolidated under a common interface
+        then this override can be removed.
+        """
+        return None

airbyte_cdk/sources/file_based/stream/concurrent/adapters.py

@@ -4,10 +4,19 @@
 
 import copy
 import logging
-from functools import lru_cache
+from functools import cache, lru_cache
 from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteStream, Level, SyncMode, Type
+from typing_extensions import deprecated
+
+from airbyte_cdk.models import (
+    AirbyteLogMessage,
+    AirbyteMessage,
+    ConfiguredAirbyteStream,
+    Level,
+    SyncMode,
+    Type,
+)
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.file_based.availability_strategy import (
@@ -22,27 +31,35 @@ from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedFin
 from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
 from airbyte_cdk.sources.file_based.types import StreamSlice
 from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.source import ExperimentalClassWarning
 from airbyte_cdk.sources.streams.concurrent.abstract_stream_facade import AbstractStreamFacade
 from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage
-from airbyte_cdk.sources.streams.concurrent.helpers import get_cursor_field_from_stream, get_primary_key_from_stream
+from airbyte_cdk.sources.streams.concurrent.helpers import (
+    get_cursor_field_from_stream,
+    get_primary_key_from_stream,
+)
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
 from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
-from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
 from airbyte_cdk.sources.streams.core import StreamData
+from airbyte_cdk.sources.types import Record
 from airbyte_cdk.sources.utils.schema_helpers import InternalConfig
 from airbyte_cdk.sources.utils.slice_logger import SliceLogger
-from deprecated.classic import deprecated
 
 if TYPE_CHECKING:
-    from airbyte_cdk.sources.file_based.stream.concurrent.cursor import AbstractConcurrentFileBasedCursor
+    from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
+        AbstractConcurrentFileBasedCursor,
+    )
 
 """
 This module contains adapters to help enabling concurrency on File-based Stream objects without needing to migrate to AbstractStream
 """
 
 
-@deprecated("This class is experimental. Use at your own risk.")
+@deprecated(
+    "This class is experimental. Use at your own risk.",
+    category=ExperimentalClassWarning,
+)
 class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBasedStream):
     @classmethod
     def create_from_stream(
@@ -71,7 +88,9 @@ class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBas
                 partition_generator=FileBasedStreamPartitionGenerator(
                     stream,
                     message_repository,
-                    SyncMode.full_refresh if isinstance(cursor, FileBasedFinalStateCursor) else SyncMode.incremental,
+                    SyncMode.full_refresh
+                    if isinstance(cursor, FileBasedFinalStateCursor)
+                    else SyncMode.incremental,
                     [cursor_field] if cursor_field is not None else None,
                     state,
                     cursor,
@@ -127,6 +146,7 @@ class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBas
         return self._legacy_stream.supports_incremental
 
     @property
+    @deprecated("Deprecated as of CDK version 3.7.0.")
     def availability_strategy(self) -> AbstractFileBasedAvailabilityStrategy:
         return self._legacy_stream.availability_strategy
 
@@ -136,7 +156,10 @@ class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBas
 
     @property
     def primary_key(self) -> PrimaryKeyType:
-        return self._legacy_stream.config.primary_key or self.get_parser().get_parser_defined_primary_key(self._legacy_stream.config)
+        return (
+            self._legacy_stream.config.primary_key
+            or self.get_parser().get_parser_defined_primary_key(self._legacy_stream.config)
+        )
 
     def get_parser(self) -> FileTypeParser:
         return self._legacy_stream.get_parser()
@@ -145,7 +168,7 @@ class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBas
         return self._legacy_stream.get_files()
 
     def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[Mapping[str, Any]]:
-        yield from self._legacy_stream.read_records_from_slice(stream_slice)
+        yield from self._legacy_stream.read_records_from_slice(stream_slice)  # type: ignore[misc] # Only Mapping[str, Any] is expected for legacy streams, not AirbyteMessage
 
     def compute_slices(self) -> Iterable[Optional[StreamSlice]]:
         return self._legacy_stream.compute_slices()
@@ -183,7 +206,10 @@ class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBas
                 # This shouldn't happen if the ConcurrentCursor was used
                 state = "unknown; no state attribute was available on the cursor"
             yield AirbyteMessage(
-                type=Type.LOG, log=AirbyteLogMessage(level=Level.ERROR, message=f"Cursor State at time of exception: {state}")
+                type=Type.LOG,
+                log=AirbyteLogMessage(
+                    level=Level.ERROR, message=f"Cursor State at time of exception: {state}"
+                ),
             )
             raise exc
 
@@ -204,7 +230,6 @@ class FileBasedStreamPartition(Partition):
         sync_mode: SyncMode,
         cursor_field: Optional[List[str]],
         state: Optional[MutableMapping[str, Any]],
-        cursor: "AbstractConcurrentFileBasedCursor",
     ):
         self._stream = stream
         self._slice = _slice
@@ -212,8 +237,6 @@ class FileBasedStreamPartition(Partition):
         self._sync_mode = sync_mode
         self._cursor_field = cursor_field
         self._state = state
-        self._cursor = cursor
-        self._is_closed = False
 
     def read(self) -> Iterable[Record]:
         try:
@@ -225,11 +248,30 @@ class FileBasedStreamPartition(Partition):
             ):
                 if isinstance(record_data, Mapping):
                     data_to_return = dict(record_data)
-                    self._stream.transformer.transform(data_to_return, self._stream.get_json_schema())
-                    yield Record(data_to_return, self.stream_name())
-                elif isinstance(record_data, AirbyteMessage) and record_data.type == Type.RECORD:
+                    self._stream.transformer.transform(
+                        data_to_return, self._stream.get_json_schema()
+                    )
+                    yield Record(data=data_to_return, stream_name=self.stream_name())
+                elif (
+                    isinstance(record_data, AirbyteMessage)
+                    and record_data.type == Type.RECORD
+                    and record_data.record is not None
+                ):
                     # `AirbyteMessage`s of type `Record` should also be yielded so they are enqueued
-                    yield Record(record_data.record.data, self.stream_name())
+                    # If stream is flagged for file_transfer the record should data in file key
+                    record_message_data = (
+                        record_data.record.file
+                        if self._use_file_transfer()
+                        else record_data.record.data
+                    )
+                    if not record_message_data:
+                        raise ExceptionWithDisplayMessage("A record without data was found")
+                    else:
+                        yield Record(
+                            data=record_message_data,
+                            stream_name=self.stream_name(),
+                            is_file_transfer_message=self._use_file_transfer(),
+                        )
                 else:
                     self._message_repository.emit_message(record_data)
         except Exception as e:
@@ -248,13 +290,6 @@ class FileBasedStreamPartition(Partition):
         file = self._slice["files"][0]
         return {"files": [file]}
 
-    def close(self) -> None:
-        self._cursor.close_partition(self)
-        self._is_closed = True
-
-    def is_closed(self) -> bool:
-        return self._is_closed
-
     def __hash__(self) -> int:
         if self._slice:
             # Convert the slice to a string so that it can be hashed
@@ -271,6 +306,10 @@ class FileBasedStreamPartition(Partition):
     def stream_name(self) -> str:
         return self._stream.name
 
+    @cache
+    def _use_file_transfer(self) -> bool:
+        return hasattr(self._stream, "use_file_transfer") and self._stream.use_file_transfer
+
     def __repr__(self) -> str:
         return f"FileBasedStreamPartition({self._stream.name}, {self._slice})"
 
@@ -294,7 +333,9 @@ class FileBasedStreamPartitionGenerator(PartitionGenerator):
 
     def generate(self) -> Iterable[FileBasedStreamPartition]:
         pending_partitions = []
-        for _slice in self._stream.stream_slices(sync_mode=self._sync_mode, cursor_field=self._cursor_field, stream_state=self._state):
+        for _slice in self._stream.stream_slices(
+            sync_mode=self._sync_mode, cursor_field=self._cursor_field, stream_state=self._state
+        ):
             if _slice is not None:
                 for file in _slice.get("files", []):
                     pending_partitions.append(
@@ -305,7 +346,6 @@ class FileBasedStreamPartitionGenerator(PartitionGenerator):
                             self._sync_mode,
                             self._cursor_field,
                             self._state,
-                            self._cursor,
                         )
                     )
         self._cursor.set_pending_partitions(pending_partitions)

airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py

@@ -2,4 +2,8 @@ from .abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCu
 from .file_based_concurrent_cursor import FileBasedConcurrentCursor
 from .file_based_final_state_cursor import FileBasedFinalStateCursor
 
-__all__ = ["AbstractConcurrentFileBasedCursor", "FileBasedConcurrentCursor", "FileBasedFinalStateCursor"]
+__all__ = [
+    "AbstractConcurrentFileBasedCursor",
+    "FileBasedConcurrentCursor",
+    "FileBasedFinalStateCursor",
+]

airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py

@@ -12,7 +12,7 @@ from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
 from airbyte_cdk.sources.file_based.types import StreamState
 from airbyte_cdk.sources.streams.concurrent.cursor import Cursor
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
-from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
+from airbyte_cdk.sources.types import Record
 
 if TYPE_CHECKING:
     from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition
@@ -24,45 +24,36 @@ class AbstractConcurrentFileBasedCursor(Cursor, AbstractFileBasedCursor, ABC):
 
     @property
     @abstractmethod
-    def state(self) -> MutableMapping[str, Any]:
-        ...
+    def state(self) -> MutableMapping[str, Any]: ...
 
     @abstractmethod
-    def observe(self, record: Record) -> None:
-        ...
+    def observe(self, record: Record) -> None: ...
 
     @abstractmethod
-    def close_partition(self, partition: Partition) -> None:
-        ...
+    def close_partition(self, partition: Partition) -> None: ...
 
     @abstractmethod
-    def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None:
-        ...
+    def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None: ...
 
     @abstractmethod
-    def add_file(self, file: RemoteFile) -> None:
-        ...
+    def add_file(self, file: RemoteFile) -> None: ...
 
     @abstractmethod
-    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
-        ...
+    def get_files_to_sync(
+        self, all_files: Iterable[RemoteFile], logger: logging.Logger
+    ) -> Iterable[RemoteFile]: ...
 
     @abstractmethod
-    def get_state(self) -> MutableMapping[str, Any]:
-        ...
+    def get_state(self) -> MutableMapping[str, Any]: ...
 
     @abstractmethod
-    def set_initial_state(self, value: StreamState) -> None:
-        ...
+    def set_initial_state(self, value: StreamState) -> None: ...
 
     @abstractmethod
-    def get_start_time(self) -> datetime:
-        ...
+    def get_start_time(self) -> datetime: ...
 
     @abstractmethod
-    def emit_state_message(self) -> None:
-        ...
+    def emit_state_message(self) -> None: ...
 
     @abstractmethod
-    def ensure_at_least_one_state_emitted(self) -> None:
-        ...
+    def ensure_at_least_one_state_emitted(self) -> None: ...