airbyte-cdk 0.61.2__py3-none-any.whl → 0.62.1__py3-none-any.whl

airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py

@@ -0,0 +1,279 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from datetime import datetime, timedelta
+from threading import RLock
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, MutableMapping, Optional, Tuple
+
+from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
+from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
+from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor
+from airbyte_cdk.sources.file_based.types import StreamState
+from airbyte_cdk.sources.message.repository import MessageRepository
+from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
+from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
+from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
+
+if TYPE_CHECKING:
+    from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition
+
+_NULL_FILE = ""
+
+
+class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
+    CURSOR_FIELD = "_ab_source_file_last_modified"
+    DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL = DefaultFileBasedCursor.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
+    DEFAULT_MAX_HISTORY_SIZE = 10_000
+    DATE_TIME_FORMAT = DefaultFileBasedCursor.DATE_TIME_FORMAT
+    zero_value = datetime.min
+    zero_cursor_value = f"0001-01-01T00:00:00.000000Z_{_NULL_FILE}"
+
+    def __init__(
+        self,
+        stream_config: FileBasedStreamConfig,
+        stream_name: str,
+        stream_namespace: Optional[str],
+        stream_state: MutableMapping[str, Any],
+        message_repository: MessageRepository,
+        connector_state_manager: ConnectorStateManager,
+        cursor_field: CursorField,
+    ) -> None:
+        super().__init__()
+        self._stream_name = stream_name
+        self._stream_namespace = stream_namespace
+        self._state = stream_state
+        self._message_repository = message_repository
+        self._connector_state_manager = connector_state_manager
+        self._cursor_field = cursor_field
+        self._time_window_if_history_is_full = timedelta(
+            days=stream_config.days_to_sync_if_history_is_full or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
+        )
+        self._state_lock = RLock()
+        self._pending_files_lock = RLock()
+        self._pending_files: Optional[Dict[str, RemoteFile]] = None
+        self._file_to_datetime_history = stream_state.get("history", {}) if stream_state else {}
+        self._prev_cursor_value = self._compute_prev_sync_cursor(stream_state)
+        self._sync_start = self._compute_start_time()
+
+    @property
+    def state(self) -> MutableMapping[str, Any]:
+        return self._state
+
+    def observe(self, record: Record) -> None:
+        pass
+
+    def close_partition(self, partition: Partition) -> None:
+        with self._pending_files_lock:
+            if self._pending_files is None:
+                raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.")
+
+    def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None:
+        with self._pending_files_lock:
+            self._pending_files = {}
+            for partition in partitions:
+                _slice = partition.to_slice()
+                if _slice is None:
+                    continue
+                for file in _slice["files"]:
+                    if file.uri in self._pending_files.keys():
+                        raise RuntimeError(f"Already found file {_slice} in pending files. This is unexpected. Please contact Support.")
+                self._pending_files.update({file.uri: file})
+
+    def _compute_prev_sync_cursor(self, value: Optional[StreamState]) -> Tuple[datetime, str]:
+        if not value:
+            return self.zero_value, ""
+        prev_cursor_str = value.get(self._cursor_field.cursor_field_key) or self.zero_cursor_value
+        # So if we see a cursor greater than the earliest file, it means that we have likely synced all files.
+        # However, we take the earliest file as the cursor value for the purpose of checking which files to
+        # sync, in case new files have been uploaded in the meantime.
+        # This should be very rare, as it would indicate a race condition where a file with an earlier
+        # last_modified time was uploaded after a file with a later last_modified time. Since last_modified
+        # represents the start time that the file was uploaded, we can usually expect that all previous
+        # files have already been uploaded. If that's the case, they'll be in history and we'll skip
+        # re-uploading them.
+        earliest_file_cursor_value = self._get_cursor_key_from_file(self._compute_earliest_file_in_history())
+        cursor_str = min(prev_cursor_str, earliest_file_cursor_value)
+        cursor_dt, cursor_uri = cursor_str.split("_", 1)
+        return datetime.strptime(cursor_dt, self.DATE_TIME_FORMAT), cursor_uri
+
+    def _get_cursor_key_from_file(self, file: Optional[RemoteFile]) -> str:
+        if file:
+            return f"{datetime.strftime(file.last_modified, self.DATE_TIME_FORMAT)}_{file.uri}"
+        return self.zero_cursor_value
+
+    def _compute_earliest_file_in_history(self) -> Optional[RemoteFile]:
+        with self._state_lock:
+            if self._file_to_datetime_history:
+                filename, last_modified = min(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
+                return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
+            else:
+                return None
+
+    def add_file(self, file: RemoteFile) -> None:
+        """
+        Add a file to the cursor. This method is called when a file is processed by the stream.
+        :param file: The file to add
+        """
+        if self._pending_files is None:
+            raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.")
+        with self._pending_files_lock:
+            with self._state_lock:
+                if file.uri not in self._pending_files:
+                    self._message_repository.emit_message(
+                        AirbyteMessage(
+                            type=Type.LOG,
+                            log=AirbyteLogMessage(
+                                level=Level.WARN,
+                                message=f"The file {file.uri} was not found in the list of pending files. This is unexpected. Please contact Support",
+                            ),
+                        )
+                    )
+                else:
+                    self._pending_files.pop(file.uri)
+                self._file_to_datetime_history[file.uri] = file.last_modified.strftime(self.DATE_TIME_FORMAT)
+                if len(self._file_to_datetime_history) > self.DEFAULT_MAX_HISTORY_SIZE:
+                    # Get the earliest file based on its last modified date and its uri
+                    oldest_file = self._compute_earliest_file_in_history()
+                    if oldest_file:
+                        del self._file_to_datetime_history[oldest_file.uri]
+                    else:
+                        raise Exception(
+                            "The history is full but there is no files in the history. This should never happen and might be indicative of a bug in the CDK."
+                        )
+                self.emit_state_message()
+
+    def emit_state_message(self) -> None:
+        with self._state_lock:
+            new_state = self.get_state()
+            self._connector_state_manager.update_state_for_stream(
+                self._stream_name,
+                self._stream_namespace,
+                new_state,
+            )
+            state_message = self._connector_state_manager.create_state_message(
+                self._stream_name, self._stream_namespace, send_per_stream_state=True
+            )
+            self._message_repository.emit_message(state_message)
+
+    def _get_new_cursor_value(self) -> str:
+        with self._pending_files_lock:
+            with self._state_lock:
+                if self._pending_files:
+                    # If there are partitions that haven't been synced, we don't know whether the files that have been synced
+                    # represent a contiguous region.
+                    # To avoid missing files, we only increment the cursor up to the oldest pending file, because we know
+                    # that all older files have been synced.
+                    return self._get_cursor_key_from_file(self._compute_earliest_pending_file())
+                elif self._file_to_datetime_history:
+                    # If all partitions have been synced, we know that the sync is up-to-date and so can advance
+                    # the cursor to the newest file in history.
+                    return self._get_cursor_key_from_file(self._compute_latest_file_in_history())
+                else:
+                    return f"{self.zero_value.strftime(self.DATE_TIME_FORMAT)}_"
+
+    def _compute_earliest_pending_file(self) -> Optional[RemoteFile]:
+        if self._pending_files:
+            return min(self._pending_files.values(), key=lambda x: x.last_modified)
+        else:
+            return None
+
+    def _compute_latest_file_in_history(self) -> Optional[RemoteFile]:
+        with self._state_lock:
+            if self._file_to_datetime_history:
+                filename, last_modified = max(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
+                return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
+            else:
+                return None
+
+    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+        """
+        Given the list of files in the source, return the files that should be synced.
+        :param all_files: All files in the source
+        :param logger:
+        :return: The files that should be synced
+        """
+        with self._state_lock:
+            if self._is_history_full():
+                logger.warning(
+                    f"The state history is full. "
+                    f"This sync and future syncs won't be able to use the history to filter out duplicate files. "
+                    f"It will instead use the time window of {self._time_window_if_history_is_full} to filter out files."
+                )
+            for f in all_files:
+                if self._should_sync_file(f, logger):
+                    yield f
+
+    def _should_sync_file(self, file: RemoteFile, logger: logging.Logger) -> bool:
+        with self._state_lock:
+            if file.uri in self._file_to_datetime_history:
+                # If the file's uri is in the history, we should sync the file if it has been modified since it was synced
+                updated_at_from_history = datetime.strptime(self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT)
+                if file.last_modified < updated_at_from_history:
+                    self._message_repository.emit_message(
+                        AirbyteMessage(
+                            type=Type.LOG,
+                            log=AirbyteLogMessage(
+                                level=Level.WARN,
+                                message=f"The file {file.uri}'s last modified date is older than the last time it was synced. This is unexpected. Skipping the file.",
+                            ),
+                        )
+                    )
+                    return False
+                else:
+                    return file.last_modified > updated_at_from_history
+
+            prev_cursor_timestamp, prev_cursor_uri = self._prev_cursor_value
+            if self._is_history_full():
+                if file.last_modified > prev_cursor_timestamp:
+                    # If the history is partial and the file's datetime is strictly greater than the cursor, we should sync it
+                    return True
+                elif file.last_modified == prev_cursor_timestamp:
+                    # If the history is partial and the file's datetime is equal to the earliest file in the history,
+                    # we should sync it if its uri is greater than or equal to the cursor value.
+                    return file.uri > prev_cursor_uri
+                else:
+                    return file.last_modified >= self._sync_start
+            else:
+                # The file is not in the history and the history is complete. We know we need to sync the file
+                return True
+
+    def _is_history_full(self) -> bool:
+        """
+        Returns true if the state's history is full, meaning new entries will start to replace old entries.
+        """
+        with self._state_lock:
+            if self._file_to_datetime_history is None:
+                raise RuntimeError("The history object has not been set. This is unexpected. Please contact Support.")
+            return len(self._file_to_datetime_history) >= self.DEFAULT_MAX_HISTORY_SIZE
+
+    def _compute_start_time(self) -> datetime:
+        if not self._file_to_datetime_history:
+            return datetime.min
+        else:
+            earliest = min(self._file_to_datetime_history.values())
+            earliest_dt = datetime.strptime(earliest, self.DATE_TIME_FORMAT)
+            if self._is_history_full():
+                time_window = datetime.now() - self._time_window_if_history_is_full
+                earliest_dt = min(earliest_dt, time_window)
+            return earliest_dt
+
+    def get_start_time(self) -> datetime:
+        return self._sync_start
+
+    def get_state(self) -> MutableMapping[str, Any]:
+        """
+        Get the state of the cursor.
+        """
+        with self._state_lock:
+            return {"history": self._file_to_datetime_history, self._cursor_field.cursor_field_key: self._get_new_cursor_value()}
+
+    def set_initial_state(self, value: StreamState) -> None:
+        pass
+
+    def ensure_at_least_one_state_emitted(self) -> None:
+        self.emit_state_message()

airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Iterable, List, MutableMapping
+
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
+from airbyte_cdk.sources.file_based.types import StreamState
+from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
+from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
+
+if TYPE_CHECKING:
+    from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition
+
+
+class FileBasedNoopCursor(AbstractConcurrentFileBasedCursor):
+    def __init__(self, stream_config: FileBasedStreamConfig, **kwargs: Any):
+        pass
+
+    @property
+    def state(self) -> MutableMapping[str, Any]:
+        return {}
+
+    def observe(self, record: Record) -> None:
+        pass
+
+    def close_partition(self, partition: Partition) -> None:
+        pass
+
+    def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None:
+        pass
+
+    def add_file(self, file: RemoteFile) -> None:
+        pass
+
+    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+        return all_files
+
+    def get_state(self) -> MutableMapping[str, Any]:
+        return {}
+
+    def set_initial_state(self, value: StreamState) -> None:
+        return None
+
+    def get_start_time(self) -> datetime:
+        return datetime.min
+
+    def emit_state_message(self) -> None:
+        pass
+
+    def ensure_at_least_one_state_emitted(self) -> None:
+        pass

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -43,9 +43,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     ab_file_name_col = "_ab_source_file_url"
     airbyte_columns = [ab_last_mod_col, ab_file_name_col]
 
-    def __init__(self, cursor: AbstractFileBasedCursor, **kwargs: Any):
+    def __init__(self, **kwargs: Any):
         super().__init__(**kwargs)
-        self._cursor = cursor
 
     @property
     def state(self) -> MutableMapping[str, Any]:
@@ -56,6 +55,16 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         """State setter, accept state serialized by state getter."""
         self._cursor.set_initial_state(value)
 
+    @property
+    def cursor(self) -> Optional[AbstractFileBasedCursor]:
+        return self._cursor
+
+    @cursor.setter
+    def cursor(self, value: AbstractFileBasedCursor) -> None:
+        if self._cursor is not None:
+            raise RuntimeError(f"Cursor for stream {self.name} is already set. This is unexpected. Please contact Support.")
+        self._cursor = value
+
     @property
     def primary_key(self) -> PrimaryKeyType:
         return self.config.primary_key or self.get_parser().get_parser_defined_primary_key(self.config)

airbyte_cdk/test/mock_http/mocker.py

@@ -60,7 +60,9 @@ class HttpMocker(contextlib.ContextDecorator):
         getattr(self._mocker, method)(
             requests_mock.ANY,
             additional_matcher=self._matches_wrapper(matcher),
-            response_list=[{"text": response.body, "status_code": response.status_code} for response in responses],
+            response_list=[
+                {"text": response.body, "status_code": response.status_code, "headers": response.headers} for response in responses
+            ],
         )
 
     def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:

airbyte_cdk/test/mock_http/response.py

@@ -1,10 +1,14 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
+from types import MappingProxyType
+from typing import Mapping
+
 
 class HttpResponse:
-    def __init__(self, body: str, status_code: int = 200):
+    def __init__(self, body: str, status_code: int = 200, headers: Mapping[str, str] = MappingProxyType({})):
         self._body = body
         self._status_code = status_code
+        self._headers = headers
 
     @property
     def body(self) -> str:
@@ -13,3 +17,7 @@ class HttpResponse:
     @property
     def status_code(self) -> int:
         return self._status_code
+
+    @property
+    def headers(self) -> Mapping[str, str]:
+        return self._headers

airbyte_cdk/utils/traced_exception.py

@@ -13,7 +13,6 @@ from airbyte_cdk.models import (
     AirbyteTraceMessage,
     FailureType,
     Status,
-    StreamDescriptor,
     TraceType,
 )
 from airbyte_cdk.models import Type as MessageType
@@ -44,7 +43,7 @@ class AirbyteTracedException(Exception):
         self._exception = exception
         super().__init__(internal_message)
 
-    def as_airbyte_message(self, stream_descriptor: StreamDescriptor = None) -> AirbyteMessage:
+    def as_airbyte_message(self) -> AirbyteMessage:
         """
         Builds an AirbyteTraceMessage from the exception
         """
@@ -61,7 +60,6 @@ class AirbyteTracedException(Exception):
                 internal_message=self.internal_message,
                 failure_type=self.failure_type,
                 stack_trace=stack_trace_str,
-                stream_descriptor=stream_descriptor,
             ),
         )
 
@@ -90,16 +88,3 @@ class AirbyteTracedException(Exception):
         :param exc: the exception that caused the error
         """
         return cls(internal_message=str(exc), exception=exc, *args, **kwargs)  # type: ignore  # ignoring because of args and kwargs
-
-    def as_sanitized_airbyte_message(self, stream_descriptor: StreamDescriptor = None) -> AirbyteMessage:
-        """
-        Builds an AirbyteTraceMessage from the exception and sanitizes any secrets from the message body
-        """
-        error_message = self.as_airbyte_message(stream_descriptor=stream_descriptor)
-        if error_message.trace.error.message:
-            error_message.trace.error.message = filter_secrets(error_message.trace.error.message)
-        if error_message.trace.error.internal_message:
-            error_message.trace.error.internal_message = filter_secrets(error_message.trace.error.internal_message)
-        if error_message.trace.error.stack_trace:
-            error_message.trace.error.stack_trace = filter_secrets(error_message.trace.error.stack_trace)
-        return error_message

{airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.61.2
+Version: 0.62.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.61.2.dist-info → airbyte_cdk-0.62.1.dist-info}/RECORD

@@ -24,9 +24,9 @@ airbyte_cdk/models/__init__.py,sha256=Kg8YHBqUsNWHlAw-u3ZGdG4dxLh7qBlHhqMRfamNCR
 airbyte_cdk/models/airbyte_protocol.py,sha256=DoJvnmGM3xMAZFTwA6_RGMiKSFqfE3ib_Ru0KJ65Ag4,100
 airbyte_cdk/models/well_known_types.py,sha256=KKfNbow2gdLoC1Z4hcXy_JR8m_acsB2ol7gQuEgjobw,117
 airbyte_cdk/sources/__init__.py,sha256=Ov7Uf03KPSZUmMZqZfUAK3tQwsdKjDQUDvTb-H0JyfA,1141
-airbyte_cdk/sources/abstract_source.py,sha256=Ehdh0AFBPBS8sLM7Ez7DANFsytv0tRvoNkEgr4gAV1s,15872
+airbyte_cdk/sources/abstract_source.py,sha256=GSpNwbwJ0v-KvxWa0u_nWeC0r6G2fZNkpKUhXzf6YlI,14399
 airbyte_cdk/sources/config.py,sha256=PYsY7y2u3EUwxLiEb96JnuKwH_E8CuxKggsRO2ZPSRc,856
-airbyte_cdk/sources/connector_state_manager.py,sha256=wsmUgII398MazCTKxwLBLzeiU6Z-tMTrKX882EEy-YE,10904
+airbyte_cdk/sources/connector_state_manager.py,sha256=p9iwWbb5uqRbsrHsdZBMXKmyHgLVbsOcV3QQexBFnPE,11052
 airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
 airbyte_cdk/sources/http_logger.py,sha256=v0kkpDtA0GUOgj6_3AayrYaBrSHBqG4t3MGbrtxaNmU,1437
 airbyte_cdk/sources/source.py,sha256=dk50z8Roc28MJ8FxWe652B-GwItO__bTZqFm7WOtHnw,4412
@@ -153,7 +153,7 @@ airbyte_cdk/sources/embedded/runner.py,sha256=kZ0CcUANuMjdZ4fmvp_w9P2IcsS9WSHxNq
 airbyte_cdk/sources/embedded/tools.py,sha256=-Z4tZ4AP1OTi_zrqFM3YV8Rt7c60wvsrv0Dc-rTZ2uw,744
 airbyte_cdk/sources/file_based/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/file_based/exceptions.py,sha256=-SjdDk-mbkp5qQVUESkn788W8NmGtC2LROkZRKS_Dxc,5613
-airbyte_cdk/sources/file_based/file_based_source.py,sha256=Hs9q27db2x5wnvmo7uN8NVM0EGp-7KK_R4IjR2ig3tU,11316
+airbyte_cdk/sources/file_based/file_based_source.py,sha256=WH-BCGu5vWB0WjAI54Q30uHSkX9J5QgIpBwWI5Gx7ag,13645
 airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=K9fFHcSL4E8v-X2l38wRAcZCjpyifr35orvby8vQt84,3749
 airbyte_cdk/sources/file_based/remote_file.py,sha256=dtRX7X06Fug-XDz93a5lkwPQy5nQgxH0-ZcXW2HuMGI,312
 airbyte_cdk/sources/file_based/schema_helpers.py,sha256=XBkOutIw_n6SNYU34qbyTbl0Ppt0i4k3sVFMSaX3wJo,9103
@@ -183,11 +183,14 @@ airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py,sha256=sEV
 airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py,sha256=uwk6Ugf23xKG4PRPVVRVwpcNjTwPgxejl03vLSEzK0s,604
 airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py,sha256=ZeAa0z50ywMU2chNjQ7JpL4yePU1NajhBa8FS7rXLVo,1643
 airbyte_cdk/sources/file_based/stream/__init__.py,sha256=QPDqdgjsabOQD93dSFqHGaFS_3pIwm-chEabZHiPJi0,265
-airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=GJu02B2_fcfaOnSBvhpRyXIEEtu4v8ubFR_vQpe-YAU,6405
-airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=x9VsiWhr1LBW2AEYIJYHdMZ4V_Jv2Pimll6OcTve3S8,12801
+airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=cmO1SQt5PIQRNNoh2KBv6aeY8NEY9x2dlmiRwGwU1vg,6557
+airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=qS0DJzXlVew6armFDJ0eNcSxRCmkA7JWQYFl6gcv3dU,13113
 airbyte_cdk/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=YUiyt7oMOt5P6rQalS1vJcpLnBRmgqyRgbN0HJ1ppIk,12871
-airbyte_cdk/sources/file_based/stream/concurrent/cursor.py,sha256=X8pLsNQHpmQlInpgNpwPJAYHtcgu_Xi6u0gWN-OxMK0,2650
+airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=AX_H5cIWkWOJkhXGuTSuZ56Jr5szoNfQ3NdabbWPTtI,13043
+airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py,sha256=WKEYXZwSla6xwp7k1mnyG3kl9xCzEZ9B3eE-cxIuzIM,310
+airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py,sha256=UYLE2A2RdV-5FaQ70naZZWY34l5AEJkIRlTH05-e_-k,1961
+airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py,sha256=jHiej28aKQJ3UmWXQxHRCK8xkzY5H0-zxQiVqFs5rAI,14389
+airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py,sha256=wblXBgNw-QLVPNOAL8DihlQBXbvPC1zCdDWMsPdZPzQ,1852
 airbyte_cdk/sources/file_based/stream/cursor/__init__.py,sha256=MhFB5hOo8sjwvCh8gangaymdg3EJWYt_72brFOZt068,191
 airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py,sha256=i-FPeK8lwCzX34GCcmvL5Yvdh8-uu7FeCVYDoFbD7IY,1920
 airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py,sha256=kuJRKgDYOGXRk0V0I8BpFxg0hGv7SfV_nBpmmn45F88,6815
@@ -250,9 +253,9 @@ airbyte_cdk/test/entrypoint_wrapper.py,sha256=uTOEYoWkYnbkooPJ4a4gZ-NEll5j1tTCAz
 airbyte_cdk/test/state_builder.py,sha256=SlKadhKVi38ZSKMeceVAxjowxsDDT9vJoG6gU4zDrQE,705
 airbyte_cdk/test/mock_http/__init__.py,sha256=uil6k-0NbUyDFZXtWw88HaS7r13i43VzA9H7hOHzZx8,322
 airbyte_cdk/test/mock_http/matcher.py,sha256=J4C8g8PkdKo4OwHWMJGYJIyrLnQpXI5gXWUtyxsxHpM,1240
-airbyte_cdk/test/mock_http/mocker.py,sha256=R-RvMufoI22TtZno8WpSIy0BVhtwoI9OvEGDL6HJU5w,6143
+airbyte_cdk/test/mock_http/mocker.py,sha256=Sb1Nnf3bVEJfiy5_IliRcyIiIPQL8esSWmm5j9u0E_E,6202
 airbyte_cdk/test/mock_http/request.py,sha256=dd_i47FOGD5iRlU23daotv2gEn5NOVqTBAqykxdG6-0,3687
-airbyte_cdk/test/mock_http/response.py,sha256=XjkouKy3eCsylZRROzl1qZs6LfjnO4C3L1hcJoqTHOs,354
+airbyte_cdk/test/mock_http/response.py,sha256=F09QGG8N3Z8fL_b0rmSKTYoKgku5yZJQCpj0Fwwxu3s,588
 airbyte_cdk/test/mock_http/response_builder.py,sha256=sc0lU_LN3wjBc4mFFV-3Y5IhYeapRdtB_-EDdHfyArA,7804
 airbyte_cdk/utils/__init__.py,sha256=qZoNqzEKhIXdN_ZfvXlIGnmiDDjCFy6BVCzzWjUZcuU,294
 airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=q3aDl8T10ufGbeqnUPqbZLxQcHdkf2kDfQK_upWzBbI,2894
@@ -266,7 +269,7 @@ airbyte_cdk/utils/oneof_option_config.py,sha256=N8EmWdYdwt0FM7fuShh6H8nj_r4KEL9t
 airbyte_cdk/utils/schema_inferrer.py,sha256=D8vFVgeK6VLcAug4YVAHfa3D29On0A_nMlwq9SPlfPI,3799
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=LGjSSk8lmBiC0GiHqxDwu_iMN6bCe05UMpz9e7nCw5E,741
 airbyte_cdk/utils/stream_status_utils.py,sha256=k7OY6AkJW8ifyh7ZYetC5Yy1nxM6Mx3apOAviCjJh80,971
-airbyte_cdk/utils/traced_exception.py,sha256=IDYvUkbgkOMjusiuP0xU65mHzl5nLDkhA3o-FvNDfjI,4336
+airbyte_cdk/utils/traced_exception.py,sha256=ChtuhSV_fkmMv8QjPBR1dV1US8uxlmVt_Myt-C2OIqQ,3396
 source_declarative_manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 source_declarative_manifest/main.py,sha256=HXzuRsRyhHwPrGU-hc4S7RrgoOoHImqkdfbmO2geBeE,1027
 unit_tests/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
@@ -279,14 +282,14 @@ unit_tests/singer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 unit_tests/singer/test_singer_helpers.py,sha256=pZV6VxJuK-3-FICNGmoGbokrA_zkaFZEd4rYZCVpSRU,1762
 unit_tests/singer/test_singer_source.py,sha256=edN_kv7dnYAdBveWdUYOs74ak0dK6p8uaX225h_ZILA,4442
 unit_tests/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unit_tests/sources/test_abstract_source.py,sha256=MM8-37yidqqbz87PVllWpnCl9leDaarM71siOIKj33A,59013
+unit_tests/sources/test_abstract_source.py,sha256=m-YcMK1DgIhJLKUHoANFPx_d6yh-zgLrU1wLNlNCuTg,52802
 unit_tests/sources/test_concurrent_source.py,sha256=3i7pSRetKSoP6LBpXyuXpWi2_VOwta_aTm_kgnDaLqk,3704
 unit_tests/sources/test_config.py,sha256=lxjeaf48pOMF4Pf3-Z1ux_tHTyjRFCdG_hpnxw3e7uQ,2839
-unit_tests/sources/test_connector_state_manager.py,sha256=ynFxA63Cxe6t-wMMh9C6ByTlMAuk8W7H2FikDhnUEQ0,24264
+unit_tests/sources/test_connector_state_manager.py,sha256=KAvYmuaWwg2kSnPNKri6Ne8TmLpsSimotsnDLLKkDD0,24369
 unit_tests/sources/test_http_logger.py,sha256=VT6DqgspI3DcRnoBQkkQX0z4dF_AOiYZ5P_zxmMW8oU,9004
-unit_tests/sources/test_integration_source.py,sha256=qcWld9evB1rAjALWX8SDshGz7seYkN3HCamQ6KQ2Idw,4269
+unit_tests/sources/test_integration_source.py,sha256=u_w5NS9n8GkTsoTjJvBE3-g8x0NG2054hL3PtW7IfAM,3458
 unit_tests/sources/test_source.py,sha256=W0I4umL_d_OToLYYiRkjkJR6e-cCYjdV8zKc3uLvF0k,27999
-unit_tests/sources/test_source_read.py,sha256=n9XpVQLfsQH8eh6D99MDiNVBBKcf6UtouThDJcGH6SU,17186
+unit_tests/sources/test_source_read.py,sha256=AEFoJfzM0_5QQIJyKwGLK_kq_Vz_CBivImnUnXJQJ0I,17176
 unit_tests/sources/concurrent_source/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py,sha256=zsGnMcEsBedjW8wahil6LNqniil-3NXhyZd5W-80Km0,3665
 unit_tests/sources/declarative/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
@@ -366,11 +369,11 @@ unit_tests/sources/declarative/states/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyR
 unit_tests/sources/declarative/stream_slicers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py,sha256=FADH2_qI8mD0K1NEl8--MJwSYiRtOqesv3msR-xnAlM,7825
 unit_tests/sources/file_based/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unit_tests/sources/file_based/helpers.py,sha256=MZTwaWtX0a6TPbFcUMP-EgqBunK2wpoElgApCEE1bN4,2659
+unit_tests/sources/file_based/helpers.py,sha256=JSKXrPL7iSBSH7nkKde-jcylVuDohJid5Oi4YtVRPwg,2854
 unit_tests/sources/file_based/in_memory_files_source.py,sha256=1UCfRMgaovPdhkORT5k5Izj6e0ldPp802iiaffG2ghk,8550
-unit_tests/sources/file_based/test_file_based_scenarios.py,sha256=rQaORUdsRdWxTMMshJxAxnp3x6Bsnuirit4yjrT0Oao,11680
+unit_tests/sources/file_based/test_file_based_scenarios.py,sha256=r2qP3WbDV48YyeBt5o18Ztt8r1v0-zZPjgrITOiJSVM,15356
 unit_tests/sources/file_based/test_file_based_stream_reader.py,sha256=P6yTp7tbPfREzi5SXg4SSSql5nxiRV571YdOmwb_SzY,9219
-unit_tests/sources/file_based/test_scenarios.py,sha256=xE5OoCcyTVMP0QFUgTiT0-qtigjlbQvqeIwFRNEWmgE,8752
+unit_tests/sources/file_based/test_scenarios.py,sha256=w7ROHOY6dcbd5-tqcf1OtdBfA-QCC7n7Y0FP9zcH4bY,9317
 unit_tests/sources/file_based/test_schema_helpers.py,sha256=IYIDdLRK41RkSG_ZW2cagAt9krV4QLbkzu6r7vPx9Js,12047
 unit_tests/sources/file_based/availability_strategy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py,sha256=14ffoRWC4RHPrmBFZpplnAd1Uezn8neuQrIyZqvjTK0,4964
@@ -389,18 +392,22 @@ unit_tests/sources/file_based/file_types/test_unstructured_parser.py,sha256=kmVl
 unit_tests/sources/file_based/scenarios/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/file_based/scenarios/avro_scenarios.py,sha256=oeQUmCV7d2aTShreYc-PvVb4cWqLSsVwHfg-lcKjzPs,30554
 unit_tests/sources/file_based/scenarios/check_scenarios.py,sha256=0xkt21ASTnTAMP0RYJEsF3yMGsNN7wWOoG_tmzL9PYw,6750
+unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py,sha256=h1k8ZZ8UODX0i6Squz5bI59ldQGqAPpxoKFkdfBjvAQ,103709
 unit_tests/sources/file_based/scenarios/csv_scenarios.py,sha256=w4yPhLQ_7flHTMiKIxRNTmv4ru6J-t2HnYxJsd_B1fg,123227
 unit_tests/sources/file_based/scenarios/file_based_source_builder.py,sha256=3gAFkguYH87v_WpV0lUttTKu7LG8a-viokDW232ecUw,4123
-unit_tests/sources/file_based/scenarios/incremental_scenarios.py,sha256=L5NTFW7S-NAHgCyP0QczgiZtgGtdzbyQO92iu45Z3Kc,67257
+unit_tests/sources/file_based/scenarios/incremental_scenarios.py,sha256=68lmzAdan9ohO7kTuKJ9eafpJhJJSehf9GzpV-lryKI,67817
 unit_tests/sources/file_based/scenarios/jsonl_scenarios.py,sha256=vE-j-8lszkbqU_7t_1OK2EBvpHKA9dJDrmKGxesVMbY,31733
 unit_tests/sources/file_based/scenarios/parquet_scenarios.py,sha256=0DZbrb2wbaGSQ3OjD8gCH673dPqtVcLCR_LVkA_qVpA,26658
-unit_tests/sources/file_based/scenarios/scenario_builder.py,sha256=mqs9q-MSwAGT6m7n5hChEygLlckGoR3lpj5ACk9UTIs,9715
+unit_tests/sources/file_based/scenarios/scenario_builder.py,sha256=ynywaMWNvPnJ8Mg2h3vYZPLfaOzHcSFYj7e8bmY_0gY,9894
 unit_tests/sources/file_based/scenarios/unstructured_scenarios.py,sha256=2_p15Phk2xiBgZ0OdGYrCU9eAlTT8h_SU5nk1ehUcLk,67894
 unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py,sha256=FVYbRfdj2RCLFVwUNqQKiBFMm78y6FvmTO447i3SXqY,28697
 unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py,sha256=pXcLV4MTtT55qHDjuf9aYHi5K3dVX0YSpltuAQApISs,28511
 unit_tests/sources/file_based/stream/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/file_based/stream/test_default_file_based_cursor.py,sha256=XhtCGvgSBFyeQwgqGciPsIB1HIlWqTcXROwnxrjutHc,13109
 unit_tests/sources/file_based/stream/test_default_file_based_stream.py,sha256=1GZPMIL00KGMIaYcPPBhQ0gpkYAJ48xtxXOgEwxkg84,10263
+unit_tests/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+unit_tests/sources/file_based/stream/concurrent/test_adapters.py,sha256=aDtXineIKv1VfJ5xQWszgBfeomSKytrYh_FEOt9DnUc,15202
+unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py,sha256=ovST4Hle8dEb-wtjx-x6DJ0VxCjvGlKWKKT9XL29nzs,19960
 unit_tests/sources/fixtures/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 unit_tests/sources/fixtures/source_test_fixture.py,sha256=dvpISgio2sOp-U3bXudH_49vY4c68sO_PMs1JZTMaj0,5502
 unit_tests/sources/message/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -421,7 +428,7 @@ unit_tests/sources/streams/concurrent/test_partition_reader.py,sha256=bBX2mP62t4
 unit_tests/sources/streams/concurrent/test_thread_pool_manager.py,sha256=l0rwdDX79MRip0IKTXKGIqEZy2NptMTUTPYYQQU5yjQ,4203
 unit_tests/sources/streams/concurrent/scenarios/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py,sha256=JlsJrYFgs2tg5hb1q47_YEPJ2_i7K_IMaHMNt9VWVdI,9910
-unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=dY3dpAru55V8iZ5z74AE8qxfLBwimR9av2UfTJMPsGg,5912
+unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py,sha256=HKtWlCbx81CdS8hqCs-d43JndiLL6Tp4K0Yf8VdycDg,6239
 unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py,sha256=eA5Kug7AAKoNtW3gBSJWDz-Y-Udi3ulwKgAxxs8skAc,13992
 unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py,sha256=Z_4-ClsxBupmN7Pbl8lF9bkSA9wnjLtrgA9WR_8VRi8,3757
 unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py,sha256=Qa1z48QLKy8xOViyiqpkIEhREF4rZHqJh8FwJ8fzqiQ,13435
@@ -438,7 +445,7 @@ unit_tests/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/test/test_entrypoint_wrapper.py,sha256=m4csYvjO2PzvZZma7K322SBBiL5D33xuv8eUMjitDXE,10839
 unit_tests/test/mock_http/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/test/mock_http/test_matcher.py,sha256=dBndYzqvo3AdHRilLrqruXdPviwi91gWt-ubDsGb-yg,2327
-unit_tests/test/mock_http/test_mocker.py,sha256=Zp7nhuEGWLsysNFMIcNhu5GhWrNd_JQMIXOjZHYRdY4,8409
+unit_tests/test/mock_http/test_mocker.py,sha256=sOoWutnrPDKB99Y3bkEyr3HFELGivwuklVJ_ii8C-ew,8523
 unit_tests/test/mock_http/test_request.py,sha256=O9ihefGNiZKpHqsGtis6BjF8VoaOULNR8zOblVqmsL4,7602
 unit_tests/test/mock_http/test_response_builder.py,sha256=IxAww4gaOxG-9MW8kEZkRzYL2mO6xe4jIsxhi40i2ow,7878
 unit_tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -449,8 +456,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
 unit_tests/utils/test_stream_status_utils.py,sha256=Xr8MZ2HWgTVIyMbywDvuYkRaUF4RZLQOT8-JjvcfR24,2970
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.61.2.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.61.2.dist-info/METADATA,sha256=rRpxOblfkKNJTkiGdS1S3AJSVoNm10I4DX1nZG3J_gw,11073
-airbyte_cdk-0.61.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-airbyte_cdk-0.61.2.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.61.2.dist-info/RECORD,,
+airbyte_cdk-0.62.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.62.1.dist-info/METADATA,sha256=9m_kiUnGlScq9ANU1dhvx00D_p5inrTOtJ-09MkF1vo,11073
+airbyte_cdk-0.62.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+airbyte_cdk-0.62.1.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.62.1.dist-info/RECORD,,

unit_tests/sources/file_based/helpers.py

@@ -15,6 +15,7 @@ from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeP
 from airbyte_cdk.sources.file_based.file_types.jsonl_parser import JsonlParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedConcurrentCursor
 from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor
 from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesStreamReader
 
@@ -61,5 +62,9 @@ class LowHistoryLimitCursor(DefaultFileBasedCursor):
     DEFAULT_MAX_HISTORY_SIZE = 3
 
 
+class LowHistoryLimitConcurrentCursor(FileBasedConcurrentCursor):
+    DEFAULT_MAX_HISTORY_SIZE = 3
+
+
 def make_remote_files(files: List[str]) -> List[RemoteFile]:
     return [RemoteFile(uri=f, last_modified=datetime.strptime("2023-06-05T03:54:07.000Z", "%Y-%m-%dT%H:%M:%S.%fZ")) for f in files]