airbyte-cdk 6.5.3rc2__py3-none-any.whl → 6.6.0__py3-none-any.whl

airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py

@@ -24,45 +24,36 @@ class AbstractConcurrentFileBasedCursor(Cursor, AbstractFileBasedCursor, ABC):
 
     @property
     @abstractmethod
-    def state(self) -> MutableMapping[str, Any]:
-        ...
+    def state(self) -> MutableMapping[str, Any]: ...
 
     @abstractmethod
-    def observe(self, record: Record) -> None:
-        ...
+    def observe(self, record: Record) -> None: ...
 
     @abstractmethod
-    def close_partition(self, partition: Partition) -> None:
-        ...
+    def close_partition(self, partition: Partition) -> None: ...
 
     @abstractmethod
-    def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None:
-        ...
+    def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None: ...
 
     @abstractmethod
-    def add_file(self, file: RemoteFile) -> None:
-        ...
+    def add_file(self, file: RemoteFile) -> None: ...
 
     @abstractmethod
-    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
-        ...
+    def get_files_to_sync(
+        self, all_files: Iterable[RemoteFile], logger: logging.Logger
+    ) -> Iterable[RemoteFile]: ...
 
     @abstractmethod
-    def get_state(self) -> MutableMapping[str, Any]:
-        ...
+    def get_state(self) -> MutableMapping[str, Any]: ...
 
     @abstractmethod
-    def set_initial_state(self, value: StreamState) -> None:
-        ...
+    def set_initial_state(self, value: StreamState) -> None: ...
 
     @abstractmethod
-    def get_start_time(self) -> datetime:
-        ...
+    def get_start_time(self) -> datetime: ...
 
     @abstractmethod
-    def emit_state_message(self) -> None:
-        ...
+    def emit_state_message(self) -> None: ...
 
     @abstractmethod
-    def ensure_at_least_one_state_emitted(self) -> None:
-        ...
+    def ensure_at_least_one_state_emitted(self) -> None: ...

airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py

@@ -11,7 +11,9 @@ from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import (
+    AbstractConcurrentFileBasedCursor,
+)
 from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor
 from airbyte_cdk.sources.file_based.types import StreamState
 from airbyte_cdk.sources.message.repository import MessageRepository
@@ -27,7 +29,9 @@ _NULL_FILE = ""
 
 class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
     CURSOR_FIELD = "_ab_source_file_last_modified"
-    DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL = DefaultFileBasedCursor.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
+    DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL = (
+        DefaultFileBasedCursor.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
+    )
     DEFAULT_MAX_HISTORY_SIZE = 10_000
     DATE_TIME_FORMAT = DefaultFileBasedCursor.DATE_TIME_FORMAT
     zero_value = datetime.min
@@ -51,7 +55,8 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         self._connector_state_manager = connector_state_manager
         self._cursor_field = cursor_field
         self._time_window_if_history_is_full = timedelta(
-            days=stream_config.days_to_sync_if_history_is_full or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
+            days=stream_config.days_to_sync_if_history_is_full
+            or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
         )
         self._state_lock = RLock()
         self._pending_files_lock = RLock()
@@ -70,7 +75,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
     def close_partition(self, partition: Partition) -> None:
         with self._pending_files_lock:
             if self._pending_files is None:
-                raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.")
+                raise RuntimeError(
+                    "Expected pending partitions to be set but it was not. This is unexpected. Please contact Support."
+                )
 
     def set_pending_partitions(self, partitions: List["FileBasedStreamPartition"]) -> None:
         with self._pending_files_lock:
@@ -81,7 +88,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
                     continue
                 for file in _slice["files"]:
                     if file.uri in self._pending_files.keys():
-                        raise RuntimeError(f"Already found file {_slice} in pending files. This is unexpected. Please contact Support.")
+                        raise RuntimeError(
+                            f"Already found file {_slice} in pending files. This is unexpected. Please contact Support."
+                        )
                 self._pending_files.update({file.uri: file})
 
     def _compute_prev_sync_cursor(self, value: Optional[StreamState]) -> Tuple[datetime, str]:
@@ -96,7 +105,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         # represents the start time that the file was uploaded, we can usually expect that all previous
         # files have already been uploaded. If that's the case, they'll be in history and we'll skip
         # re-uploading them.
-        earliest_file_cursor_value = self._get_cursor_key_from_file(self._compute_earliest_file_in_history())
+        earliest_file_cursor_value = self._get_cursor_key_from_file(
+            self._compute_earliest_file_in_history()
+        )
         cursor_str = min(prev_cursor_str, earliest_file_cursor_value)
         cursor_dt, cursor_uri = cursor_str.split("_", 1)
         return datetime.strptime(cursor_dt, self.DATE_TIME_FORMAT), cursor_uri
@@ -109,8 +120,13 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
     def _compute_earliest_file_in_history(self) -> Optional[RemoteFile]:
         with self._state_lock:
             if self._file_to_datetime_history:
-                filename, last_modified = min(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
-                return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
+                filename, last_modified = min(
+                    self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0])
+                )
+                return RemoteFile(
+                    uri=filename,
+                    last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT),
+                )
             else:
                 return None
 
@@ -120,7 +136,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         :param file: The file to add
         """
         if self._pending_files is None:
-            raise RuntimeError("Expected pending partitions to be set but it was not. This is unexpected. Please contact Support.")
+            raise RuntimeError(
+                "Expected pending partitions to be set but it was not. This is unexpected. Please contact Support."
+            )
         with self._pending_files_lock:
             with self._state_lock:
                 if file.uri not in self._pending_files:
@@ -135,7 +153,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
                     )
                 else:
                     self._pending_files.pop(file.uri)
-                self._file_to_datetime_history[file.uri] = file.last_modified.strftime(self.DATE_TIME_FORMAT)
+                self._file_to_datetime_history[file.uri] = file.last_modified.strftime(
+                    self.DATE_TIME_FORMAT
+                )
                 if len(self._file_to_datetime_history) > self.DEFAULT_MAX_HISTORY_SIZE:
                     # Get the earliest file based on its last modified date and its uri
                     oldest_file = self._compute_earliest_file_in_history()
@@ -155,7 +175,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
                 self._stream_namespace,
                 new_state,
             )
-            state_message = self._connector_state_manager.create_state_message(self._stream_name, self._stream_namespace)
+            state_message = self._connector_state_manager.create_state_message(
+                self._stream_name, self._stream_namespace
+            )
             self._message_repository.emit_message(state_message)
 
     def _get_new_cursor_value(self) -> str:
@@ -183,12 +205,19 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
     def _compute_latest_file_in_history(self) -> Optional[RemoteFile]:
         with self._state_lock:
             if self._file_to_datetime_history:
-                filename, last_modified = max(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
-                return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
+                filename, last_modified = max(
+                    self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0])
+                )
+                return RemoteFile(
+                    uri=filename,
+                    last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT),
+                )
             else:
                 return None
 
-    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+    def get_files_to_sync(
+        self, all_files: Iterable[RemoteFile], logger: logging.Logger
+    ) -> Iterable[RemoteFile]:
         """
         Given the list of files in the source, return the files that should be synced.
         :param all_files: All files in the source
@@ -210,7 +239,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         with self._state_lock:
             if file.uri in self._file_to_datetime_history:
                 # If the file's uri is in the history, we should sync the file if it has been modified since it was synced
-                updated_at_from_history = datetime.strptime(self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT)
+                updated_at_from_history = datetime.strptime(
+                    self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT
+                )
                 if file.last_modified < updated_at_from_history:
                     self._message_repository.emit_message(
                         AirbyteMessage(
@@ -246,7 +277,9 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         """
         with self._state_lock:
             if self._file_to_datetime_history is None:
-                raise RuntimeError("The history object has not been set. This is unexpected. Please contact Support.")
+                raise RuntimeError(
+                    "The history object has not been set. This is unexpected. Please contact Support."
+                )
             return len(self._file_to_datetime_history) >= self.DEFAULT_MAX_HISTORY_SIZE
 
     def _compute_start_time(self) -> datetime:
@@ -268,7 +301,10 @@ class FileBasedConcurrentCursor(AbstractConcurrentFileBasedCursor):
         Get the state of the cursor.
         """
         with self._state_lock:
-            return {"history": self._file_to_datetime_history, self._cursor_field.cursor_field_key: self._get_new_cursor_value()}
+            return {
+                "history": self._file_to_datetime_history,
+                self._cursor_field.cursor_field_key: self._get_new_cursor_value(),
+            }
 
     def set_initial_state(self, value: StreamState) -> None:
         pass

airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py

@@ -9,7 +9,9 @@ from typing import TYPE_CHECKING, Any, Iterable, List, MutableMapping, Optional
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import (
+    AbstractConcurrentFileBasedCursor,
+)
 from airbyte_cdk.sources.file_based.types import StreamState
 from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams import NO_CURSOR_STATE_KEY
@@ -24,7 +26,11 @@ class FileBasedFinalStateCursor(AbstractConcurrentFileBasedCursor):
     """Cursor that is used to guarantee at least one state message is emitted for a concurrent file-based stream."""
 
     def __init__(
-        self, stream_config: FileBasedStreamConfig, message_repository: MessageRepository, stream_namespace: Optional[str], **kwargs: Any
+        self,
+        stream_config: FileBasedStreamConfig,
+        message_repository: MessageRepository,
+        stream_namespace: Optional[str],
+        **kwargs: Any,
     ):
         self._stream_name = stream_config.name
         self._stream_namespace = stream_namespace
@@ -50,7 +56,9 @@ class FileBasedFinalStateCursor(AbstractConcurrentFileBasedCursor):
     def add_file(self, file: RemoteFile) -> None:
         pass
 
-    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+    def get_files_to_sync(
+        self, all_files: Iterable[RemoteFile], logger: logging.Logger
+    ) -> Iterable[RemoteFile]:
         return all_files
 
     def get_state(self) -> MutableMapping[str, Any]:
@@ -66,6 +74,10 @@ class FileBasedFinalStateCursor(AbstractConcurrentFileBasedCursor):
         pass
 
     def ensure_at_least_one_state_emitted(self) -> None:
-        self._connector_state_manager.update_state_for_stream(self._stream_name, self._stream_namespace, self.state)
-        state_message = self._connector_state_manager.create_state_message(self._stream_name, self._stream_namespace)
+        self._connector_state_manager.update_state_for_stream(
+            self._stream_name, self._stream_namespace, self.state
+        )
+        state_message = self._connector_state_manager.create_state_message(
+            self._stream_name, self._stream_namespace
+        )
         self._message_repository.emit_message(state_message)

airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py

@@ -54,7 +54,9 @@ class AbstractFileBasedCursor(ABC):
         ...
 
     @abstractmethod
-    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+    def get_files_to_sync(
+        self, all_files: Iterable[RemoteFile], logger: logging.Logger
+    ) -> Iterable[RemoteFile]:
         """
         Given the list of files in the source, return the files that should be synced.
         :param all_files: All files in the source

airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py

@@ -8,7 +8,9 @@ from typing import Any, Iterable, MutableMapping, Optional
 
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.file_based.stream.cursor.abstract_file_based_cursor import AbstractFileBasedCursor
+from airbyte_cdk.sources.file_based.stream.cursor.abstract_file_based_cursor import (
+    AbstractFileBasedCursor,
+)
 from airbyte_cdk.sources.file_based.types import StreamState
 
 
@@ -22,11 +24,14 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
         super().__init__(stream_config)
         self._file_to_datetime_history: MutableMapping[str, str] = {}
         self._time_window_if_history_is_full = timedelta(
-            days=stream_config.days_to_sync_if_history_is_full or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
+            days=stream_config.days_to_sync_if_history_is_full
+            or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL
         )
 
         if self._time_window_if_history_is_full <= timedelta():
-            raise ValueError(f"days_to_sync_if_history_is_full must be a positive timedelta, got {self._time_window_if_history_is_full}")
+            raise ValueError(
+                f"days_to_sync_if_history_is_full must be a positive timedelta, got {self._time_window_if_history_is_full}"
+            )
 
         self._start_time = self._compute_start_time()
         self._initial_earliest_file_in_history: Optional[RemoteFile] = None
@@ -37,7 +42,9 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
         self._initial_earliest_file_in_history = self._compute_earliest_file_in_history()
 
     def add_file(self, file: RemoteFile) -> None:
-        self._file_to_datetime_history[file.uri] = file.last_modified.strftime(self.DATE_TIME_FORMAT)
+        self._file_to_datetime_history[file.uri] = file.last_modified.strftime(
+            self.DATE_TIME_FORMAT
+        )
         if len(self._file_to_datetime_history) > self.DEFAULT_MAX_HISTORY_SIZE:
             # Get the earliest file based on its last modified date and its uri
             oldest_file = self._compute_earliest_file_in_history()
@@ -60,7 +67,9 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
         a string joining the last-modified timestamp of the last synced file and the name of the file.
         """
         if self._file_to_datetime_history.items():
-            filename, timestamp = max(self._file_to_datetime_history.items(), key=lambda x: (x[1], x[0]))
+            filename, timestamp = max(
+                self._file_to_datetime_history.items(), key=lambda x: (x[1], x[0])
+            )
             return f"{timestamp}_{filename}"
         return None
 
@@ -73,7 +82,9 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
     def _should_sync_file(self, file: RemoteFile, logger: logging.Logger) -> bool:
         if file.uri in self._file_to_datetime_history:
             # If the file's uri is in the history, we should sync the file if it has been modified since it was synced
-            updated_at_from_history = datetime.strptime(self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT)
+            updated_at_from_history = datetime.strptime(
+                self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT
+            )
             if file.last_modified < updated_at_from_history:
                 logger.warning(
                     f"The file {file.uri}'s last modified date is older than the last time it was synced. This is unexpected. Skipping the file."
@@ -99,7 +110,9 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
             # The file is not in the history and the history is complete. We know we need to sync the file
             return True
 
-    def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]:
+    def get_files_to_sync(
+        self, all_files: Iterable[RemoteFile], logger: logging.Logger
+    ) -> Iterable[RemoteFile]:
         if self._is_history_full():
             logger.warning(
                 f"The state history is full. "
@@ -115,8 +128,12 @@ class DefaultFileBasedCursor(AbstractFileBasedCursor):
 
     def _compute_earliest_file_in_history(self) -> Optional[RemoteFile]:
         if self._file_to_datetime_history:
-            filename, last_modified = min(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0]))
-            return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT))
+            filename, last_modified = min(
+                self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0])
+            )
+            return RemoteFile(
+                uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT)
+            )
         else:
             return None
 

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -22,7 +22,12 @@ from airbyte_cdk.sources.file_based.exceptions import (
 )
 from airbyte_cdk.sources.file_based.file_types import FileTransfer
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.file_based.schema_helpers import SchemaType, file_transfer_schema, merge_schemas, schemaless_schema
+from airbyte_cdk.sources.file_based.schema_helpers import (
+    SchemaType,
+    file_transfer_schema,
+    merge_schemas,
+    schemaless_schema,
+)
 from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream
 from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
 from airbyte_cdk.sources.file_based.types import StreamSlice
@@ -33,7 +38,6 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 
 class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
-
     """
     The default file-based stream.
     """
@@ -68,18 +72,28 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
     @cursor.setter
     def cursor(self, value: AbstractFileBasedCursor) -> None:
         if self._cursor is not None:
-            raise RuntimeError(f"Cursor for stream {self.name} is already set. This is unexpected. Please contact Support.")
+            raise RuntimeError(
+                f"Cursor for stream {self.name} is already set. This is unexpected. Please contact Support."
+            )
         self._cursor = value
 
     @property
     def primary_key(self) -> PrimaryKeyType:
-        return self.config.primary_key or self.get_parser().get_parser_defined_primary_key(self.config)
+        return self.config.primary_key or self.get_parser().get_parser_defined_primary_key(
+            self.config
+        )
 
-    def _filter_schema_invalid_properties(self, configured_catalog_json_schema: Dict[str, Any]) -> Dict[str, Any]:
+    def _filter_schema_invalid_properties(
+        self, configured_catalog_json_schema: Dict[str, Any]
+    ) -> Dict[str, Any]:
         if self.use_file_transfer:
             return {
                 "type": "object",
-                "properties": {"file_path": {"type": "string"}, "file_size": {"type": "string"}, self.ab_file_name_col: {"type": "string"}},
+                "properties": {
+                    "file_path": {"type": "string"},
+                    "file_size": {"type": "string"},
+                    self.ab_file_name_col: {"type": "string"},
+                },
             }
         else:
             return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
@@ -89,16 +103,23 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         all_files = self.list_files()
         files_to_read = self._cursor.get_files_to_sync(all_files, self.logger)
         sorted_files_to_read = sorted(files_to_read, key=lambda f: (f.last_modified, f.uri))
-        slices = [{"files": list(group[1])} for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)]
+        slices = [
+            {"files": list(group[1])}
+            for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)
+        ]
         return slices
 
-    def transform_record(self, record: dict[str, Any], file: RemoteFile, last_updated: str) -> dict[str, Any]:
+    def transform_record(
+        self, record: dict[str, Any], file: RemoteFile, last_updated: str
+    ) -> dict[str, Any]:
         # adds _ab_source_file_last_modified and _ab_source_file_url to the record
         record[self.ab_last_mod_col] = last_updated
         record[self.ab_file_name_col] = file.uri
         return record
 
-    def transform_record_for_file_transfer(self, record: dict[str, Any], file: RemoteFile) -> dict[str, Any]:
+    def transform_record_for_file_transfer(
+        self, record: dict[str, Any], file: RemoteFile
+    ) -> dict[str, Any]:
         # timstamp() returns a float representing the number of seconds since the unix epoch
         record[self.modified] = int(file.last_modified.timestamp()) * 1000
         record[self.source_file_url] = file.uri
@@ -127,15 +148,21 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                     self.logger.info(f"{self.name}: {file} file-based syncing")
                     # todo: complete here the code to not rely on local parser
                     file_transfer = FileTransfer()
-                    for record in file_transfer.get_file(self.config, file, self.stream_reader, self.logger):
+                    for record in file_transfer.get_file(
+                        self.config, file, self.stream_reader, self.logger
+                    ):
                         line_no += 1
                         if not self.record_passes_validation_policy(record):
                             n_skipped += 1
                             continue
                         record = self.transform_record_for_file_transfer(record, file)
-                        yield stream_data_to_airbyte_message(self.name, record, is_file_transfer_message=True)
+                        yield stream_data_to_airbyte_message(
+                            self.name, record, is_file_transfer_message=True
+                        )
                 else:
-                    for record in parser.parse_records(self.config, file, self.stream_reader, self.logger, schema):
+                    for record in parser.parse_records(
+                        self.config, file, self.stream_reader, self.logger, schema
+                    ):
                         line_no += 1
                         if self.config.schemaless:
                             record = {"data": record}
@@ -220,7 +247,9 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         except AirbyteTracedException as ate:
             raise ate
         except Exception as exc:
-            raise SchemaInferenceError(FileBasedSourceError.SCHEMA_INFERENCE_ERROR, stream=self.name) from exc
+            raise SchemaInferenceError(
+                FileBasedSourceError.SCHEMA_INFERENCE_ERROR, stream=self.name
+            ) from exc
         else:
             return {"type": "object", "properties": {**extra_fields, **schema["properties"]}}
 
@@ -245,14 +274,20 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                 first_n_files = self.config.recent_n_files_to_read_for_schema_discovery
 
         if first_n_files == 0:
-            self.logger.warning(msg=f"No files were identified in the stream {self.name}. Setting default schema for the stream.")
+            self.logger.warning(
+                msg=f"No files were identified in the stream {self.name}. Setting default schema for the stream."
+            )
             return schemaless_schema
 
-        max_n_files_for_schema_inference = self._discovery_policy.get_max_n_files_for_schema_inference(self.get_parser())
+        max_n_files_for_schema_inference = (
+            self._discovery_policy.get_max_n_files_for_schema_inference(self.get_parser())
+        )
 
         if first_n_files > max_n_files_for_schema_inference:
             # Use the most recent files for schema inference, so we pick up schema changes during discovery.
-            self.logger.warning(msg=f"Refusing to infer schema for {first_n_files} files; using {max_n_files_for_schema_inference} files.")
+            self.logger.warning(
+                msg=f"Refusing to infer schema for {first_n_files} files; using {max_n_files_for_schema_inference} files."
+            )
             first_n_files = max_n_files_for_schema_inference
 
         files = sorted(files, key=lambda x: x.last_modified, reverse=True)[:first_n_files]
@@ -274,7 +309,9 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         """
         Return all files that belong to the stream as defined by the stream's globs.
         """
-        return self.stream_reader.get_matching_files(self.config.globs or [], self.config.legacy_prefix, self.logger)
+        return self.stream_reader.get_matching_files(
+            self.config.globs or [], self.config.legacy_prefix, self.logger
+        )
 
     def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]:
         loop = asyncio.get_event_loop()
@@ -312,25 +349,34 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         n_started, n_files = 0, len(files)
         files_iterator = iter(files)
         while pending_tasks or n_started < n_files:
-            while len(pending_tasks) <= self._discovery_policy.n_concurrent_requests and (file := next(files_iterator, None)):
+            while len(pending_tasks) <= self._discovery_policy.n_concurrent_requests and (
+                file := next(files_iterator, None)
+            ):
                 pending_tasks.add(asyncio.create_task(self._infer_file_schema(file)))
                 n_started += 1
             # Return when the first task is completed so that we can enqueue a new task as soon as the
             # number of concurrent tasks drops below the number allowed.
-            done, pending_tasks = await asyncio.wait(pending_tasks, return_when=asyncio.FIRST_COMPLETED)
+            done, pending_tasks = await asyncio.wait(
+                pending_tasks, return_when=asyncio.FIRST_COMPLETED
+            )
             for task in done:
                 try:
                     base_schema = merge_schemas(base_schema, task.result())
                 except AirbyteTracedException as ate:
                     raise ate
                 except Exception as exc:
-                    self.logger.error(f"An error occurred inferring the schema. \n {traceback.format_exc()}", exc_info=exc)
+                    self.logger.error(
+                        f"An error occurred inferring the schema. \n {traceback.format_exc()}",
+                        exc_info=exc,
+                    )
 
         return base_schema
 
     async def _infer_file_schema(self, file: RemoteFile) -> SchemaType:
         try:
-            return await self.get_parser().infer_schema(self.config, file, self.stream_reader, self.logger)
+            return await self.get_parser().infer_schema(
+                self.config, file, self.stream_reader, self.logger
+            )
         except AirbyteTracedException as ate:
             raise ate
         except Exception as exc:

airbyte_cdk/sources/http_logger.py

@@ -9,7 +9,11 @@ from airbyte_cdk.sources.message import LogMessage
 
 
 def format_http_message(
-    response: requests.Response, title: str, description: str, stream_name: Optional[str], is_auxiliary: bool = None
+    response: requests.Response,
+    title: str,
+    description: str,
+    stream_name: Optional[str],
+    is_auxiliary: bool = None,
 ) -> LogMessage:
     request = response.request
     log_message = {

airbyte_cdk/sources/message/repository.py

@@ -28,7 +28,9 @@ _SEVERITY_BY_LOG_LEVEL = {
 
 def _is_severe_enough(threshold: Level, level: Level) -> bool:
     if threshold not in _SEVERITY_BY_LOG_LEVEL:
-        _LOGGER.warning(f"Log level {threshold} for threshold is not supported. This is probably a CDK bug. Please contact Airbyte.")
+        _LOGGER.warning(
+            f"Log level {threshold} for threshold is not supported. This is probably a CDK bug. Please contact Airbyte."
+        )
         return True
 
     if level not in _SEVERITY_BY_LOG_LEVEL:
@@ -80,7 +82,12 @@ class InMemoryMessageRepository(MessageRepository):
     def log_message(self, level: Level, message_provider: Callable[[], LogMessage]) -> None:
         if _is_severe_enough(self._log_level, level):
             self.emit_message(
-                AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=level, message=filter_secrets(json.dumps(message_provider()))))
+                AirbyteMessage(
+                    type=Type.LOG,
+                    log=AirbyteLogMessage(
+                        level=level, message=filter_secrets(json.dumps(message_provider()))
+                    ),
+                )
             )
 
     def consume_queue(self) -> Iterable[AirbyteMessage]:
@@ -89,7 +96,12 @@ class InMemoryMessageRepository(MessageRepository):
 
 
 class LogAppenderMessageRepositoryDecorator(MessageRepository):
-    def __init__(self, dict_to_append: LogMessage, decorated: MessageRepository, log_level: Level = Level.INFO):
+    def __init__(
+        self,
+        dict_to_append: LogMessage,
+        decorated: MessageRepository,
+        log_level: Level = Level.INFO,
+    ):
         self._dict_to_append = dict_to_append
         self._decorated = decorated
         self._log_level = log_level
@@ -106,7 +118,9 @@ class LogAppenderMessageRepositoryDecorator(MessageRepository):
     def consume_queue(self) -> Iterable[AirbyteMessage]:
         return self._decorated.consume_queue()
 
-    def _append_second_to_first(self, first: LogMessage, second: LogMessage, path: Optional[List[str]] = None) -> LogMessage:
+    def _append_second_to_first(
+        self, first: LogMessage, second: LogMessage, path: Optional[List[str]] = None
+    ) -> LogMessage:
         if path is None:
             path = []