airbyte-cdk 6.5.3rc2__py3-none-any.whl → 6.6.0__py3-none-any.whl

airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -92,7 +92,9 @@ class AbstractOauth2Authenticator(AuthBase):
 
         return payload
 
-    def _wrap_refresh_token_exception(self, exception: requests.exceptions.RequestException) -> bool:
+    def _wrap_refresh_token_exception(
+        self, exception: requests.exceptions.RequestException
+    ) -> bool:
         try:
             if exception.response is not None:
                 exception_content = exception.response.json()
@@ -102,7 +104,8 @@ class AbstractOauth2Authenticator(AuthBase):
             return False
         return (
             exception.response.status_code in self._refresh_token_error_status_codes
-            and exception_content.get(self._refresh_token_error_key) in self._refresh_token_error_values
+            and exception_content.get(self._refresh_token_error_key)
+            in self._refresh_token_error_values
         )
 
     @backoff.on_exception(
@@ -115,14 +118,20 @@ class AbstractOauth2Authenticator(AuthBase):
     )
     def _get_refresh_access_token_response(self) -> Any:
         try:
-            response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), data=self.build_refresh_request_body())
+            response = requests.request(
+                method="POST",
+                url=self.get_token_refresh_endpoint(),
+                data=self.build_refresh_request_body(),
+            )
             if response.ok:
                 response_json = response.json()
                 # Add the access token to the list of secrets so it is replaced before logging the response
                 # An argument could be made to remove the prevous access key from the list of secrets, but unmasking values seems like a security incident waiting to happen...
                 access_key = response_json.get(self.get_access_token_name())
                 if not access_key:
-                    raise Exception("Token refresh API response was missing access token {self.get_access_token_name()}")
+                    raise Exception(
+                        "Token refresh API response was missing access token {self.get_access_token_name()}"
+                    )
                 add_to_secrets(access_key)
                 self._log_response(response)
                 return response_json
@@ -136,7 +145,9 @@ class AbstractOauth2Authenticator(AuthBase):
                     raise DefaultBackoffException(request=e.response.request, response=e.response)
             if self._wrap_refresh_token_exception(e):
                 message = "Refresh token is invalid or expired. Please re-authenticate from Sources/<your source>/Settings."
-                raise AirbyteTracedException(internal_message=message, message=message, failure_type=FailureType.config_error)
+                raise AirbyteTracedException(
+                    internal_message=message, message=message, failure_type=FailureType.config_error
+                )
             raise
         except Exception as e:
             raise Exception(f"Error while refreshing access token: {e}") from e
@@ -149,7 +160,9 @@ class AbstractOauth2Authenticator(AuthBase):
         """
         response_json = self._get_refresh_access_token_response()
 
-        return response_json[self.get_access_token_name()], response_json[self.get_expires_in_name()]
+        return response_json[self.get_access_token_name()], response_json[
+            self.get_expires_in_name()
+        ]
 
     def _parse_token_expiration_date(self, value: Union[str, int]) -> pendulum.DateTime:
         """

airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py

@@ -6,9 +6,14 @@ from typing import Any, List, Mapping, Optional, Sequence, Tuple, Union
 
 import dpath
 import pendulum
-from airbyte_cdk.config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message
+from airbyte_cdk.config_observation import (
+    create_connector_config_control_message,
+    emit_configuration_as_airbyte_control_message,
+)
 from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
-from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import AbstractOauth2Authenticator
+from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import (
+    AbstractOauth2Authenticator,
+)
 
 
 class Oauth2Authenticator(AbstractOauth2Authenticator):
@@ -50,7 +55,9 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
         self._token_expiry_date_format = token_expiry_date_format
         self._token_expiry_is_time_of_expiration = token_expiry_is_time_of_expiration
         self._access_token = None
-        super().__init__(refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values)
+        super().__init__(
+            refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values
+        )
 
     def get_token_refresh_endpoint(self) -> str:
         return self._token_refresh_endpoint
@@ -153,8 +160,16 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             token_expiry_is_time_of_expiration bool: set True it if expires_in is returned as time of expiration instead of the number seconds until expiration
             message_repository (MessageRepository): the message repository used to emit logs on HTTP requests and control message on config update
         """
-        self._client_id = client_id if client_id is not None else dpath.get(connector_config, ("credentials", "client_id"))
-        self._client_secret = client_secret if client_secret is not None else dpath.get(connector_config, ("credentials", "client_secret"))
+        self._client_id = (
+            client_id
+            if client_id is not None
+            else dpath.get(connector_config, ("credentials", "client_id"))
+        )
+        self._client_secret = (
+            client_secret
+            if client_secret is not None
+            else dpath.get(connector_config, ("credentials", "client_secret"))
+        )
         self._access_token_config_path = access_token_config_path
         self._refresh_token_config_path = refresh_token_config_path
         self._token_expiry_date_config_path = token_expiry_date_config_path
@@ -204,18 +219,24 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         dpath.new(self._connector_config, self._refresh_token_config_path, new_refresh_token)
 
     def get_token_expiry_date(self) -> pendulum.DateTime:
-        expiry_date = dpath.get(self._connector_config, self._token_expiry_date_config_path, default="")
+        expiry_date = dpath.get(
+            self._connector_config, self._token_expiry_date_config_path, default=""
+        )
         return pendulum.now().subtract(days=1) if expiry_date == "" else pendulum.parse(expiry_date)
 
     def set_token_expiry_date(self, new_token_expiry_date):
-        dpath.new(self._connector_config, self._token_expiry_date_config_path, str(new_token_expiry_date))
+        dpath.new(
+            self._connector_config, self._token_expiry_date_config_path, str(new_token_expiry_date)
+        )
 
     def token_has_expired(self) -> bool:
         """Returns True if the token is expired"""
         return pendulum.now("UTC") > self.get_token_expiry_date()
 
     @staticmethod
-    def get_new_token_expiry_date(access_token_expires_in: str, token_expiry_date_format: str = None) -> pendulum.DateTime:
+    def get_new_token_expiry_date(
+        access_token_expires_in: str, token_expiry_date_format: str = None
+    ) -> pendulum.DateTime:
         if token_expiry_date_format:
             return pendulum.from_format(access_token_expires_in, token_expiry_date_format)
         else:
@@ -228,8 +249,12 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             str: The current access_token, updated if it was previously expired.
         """
         if self.token_has_expired():
-            new_access_token, access_token_expires_in, new_refresh_token = self.refresh_access_token()
-            new_token_expiry_date = self.get_new_token_expiry_date(access_token_expires_in, self._token_expiry_date_format)
+            new_access_token, access_token_expires_in, new_refresh_token = (
+                self.refresh_access_token()
+            )
+            new_token_expiry_date = self.get_new_token_expiry_date(
+                access_token_expires_in, self._token_expiry_date_format
+            )
             self.access_token = new_access_token
             self.set_refresh_token(new_refresh_token)
             self.set_token_expiry_date(new_token_expiry_date)
@@ -237,7 +262,9 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             #  Usually, a class shouldn't care about the implementation details but to keep backward compatibility where we print the
             #  message directly in the console, this is needed
             if not isinstance(self._message_repository, NoopMessageRepository):
-                self._message_repository.emit_message(create_connector_config_control_message(self._connector_config))
+                self._message_repository.emit_message(
+                    create_connector_config_control_message(self._connector_config)
+                )
             else:
                 emit_configuration_as_airbyte_control_message(self._connector_config)
         return self.access_token

airbyte_cdk/sources/streams/http/requests_native_auth/token.py

@@ -6,7 +6,9 @@ import base64
 from itertools import cycle
 from typing import List
 
-from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import (
+    AbstractHeaderAuthenticator,
+)
 
 
 class MultipleTokenAuthenticator(AbstractHeaderAuthenticator):
@@ -24,7 +26,9 @@ class MultipleTokenAuthenticator(AbstractHeaderAuthenticator):
     def token(self) -> str:
         return f"{self._auth_method} {next(self._tokens_iter)}"
 
-    def __init__(self, tokens: List[str], auth_method: str = "Bearer", auth_header: str = "Authorization"):
+    def __init__(
+        self, tokens: List[str], auth_method: str = "Bearer", auth_header: str = "Authorization"
+    ):
         self._auth_method = auth_method
         self._auth_header = auth_header
         self._tokens = tokens
@@ -65,7 +69,13 @@ class BasicHttpAuthenticator(AbstractHeaderAuthenticator):
     def token(self) -> str:
         return f"{self._auth_method} {self._token}"
 
-    def __init__(self, username: str, password: str = "", auth_method: str = "Basic", auth_header: str = "Authorization"):
+    def __init__(
+        self,
+        username: str,
+        password: str = "",
+        auth_method: str = "Basic",
+        auth_header: str = "Authorization",
+    ):
         auth_string = f"{username}:{password}".encode("utf8")
         b64_encoded = base64.b64encode(auth_string).decode("utf8")
         self._auth_header = auth_header

airbyte_cdk/sources/types.py

@@ -54,7 +54,11 @@ class Record(Mapping[str, Any]):
 
 class StreamSlice(Mapping[str, Any]):
     def __init__(
-        self, *, partition: Mapping[str, Any], cursor_slice: Mapping[str, Any], extra_fields: Optional[Mapping[str, Any]] = None
+        self,
+        *,
+        partition: Mapping[str, Any],
+        cursor_slice: Mapping[str, Any],
+        extra_fields: Optional[Mapping[str, Any]] = None,
     ) -> None:
         """
         :param partition: The partition keys representing a unique partition in the stream.

airbyte_cdk/sources/utils/record_helper.py

@@ -5,7 +5,12 @@ import time
 from collections.abc import Mapping as ABCMapping
 from typing import Any, Mapping, Optional
 
-from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteRecordMessage, AirbyteTraceMessage
+from airbyte_cdk.models import (
+    AirbyteLogMessage,
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    AirbyteTraceMessage,
+)
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
 from airbyte_cdk.sources.streams.core import StreamData
@@ -32,7 +37,9 @@ def stream_data_to_airbyte_message(
             # docs/connector-development/cdk-python/schemas.md for details.
             transformer.transform(data, schema)  # type: ignore
             if is_file_transfer_message:
-                message = AirbyteFileTransferRecordMessage(stream=stream_name, file=data, emitted_at=now_millis, data={})
+                message = AirbyteFileTransferRecordMessage(
+                    stream=stream_name, file=data, emitted_at=now_millis, data={}
+                )
             else:
                 message = AirbyteRecordMessage(stream=stream_name, data=data, emitted_at=now_millis)
             return AirbyteMessage(type=MessageType.RECORD, record=message)
@@ -41,4 +48,6 @@ def stream_data_to_airbyte_message(
         case AirbyteLogMessage():
             return AirbyteMessage(type=MessageType.LOG, log=data_or_message)
         case _:
-            raise ValueError(f"Unexpected type for data_or_message: {type(data_or_message)}: {data_or_message}")
+            raise ValueError(
+                f"Unexpected type for data_or_message: {type(data_or_message)}: {data_or_message}"
+            )

airbyte_cdk/sources/utils/schema_helpers.py

@@ -74,7 +74,9 @@ def _expand_refs(schema: Any, ref_resolver: Optional[RefResolver] = None) -> Non
         if "$ref" in schema:
             ref_url = schema.pop("$ref")
             _, definition = ref_resolver.resolve(ref_url)
-            _expand_refs(definition, ref_resolver=ref_resolver)  # expand refs in definitions as well
+            _expand_refs(
+                definition, ref_resolver=ref_resolver
+            )  # expand refs in definitions as well
             schema.update(definition)
         else:
             for key, value in schema.items():
@@ -152,7 +154,9 @@ class ResourceSchemaLoader:
             base = os.path.dirname(package.__file__) + "/"
         else:
             raise ValueError(f"Package {package} does not have a valid __file__ field")
-        resolved = jsonref.JsonRef.replace_refs(raw_schema, loader=JsonFileLoader(base, "schemas/shared"), base_uri=base)
+        resolved = jsonref.JsonRef.replace_refs(
+            raw_schema, loader=JsonFileLoader(base, "schemas/shared"), base_uri=base
+        )
         resolved = resolve_ref_links(resolved)
         if isinstance(resolved, dict):
             return resolved
@@ -160,7 +164,9 @@ class ResourceSchemaLoader:
             raise ValueError(f"Expected resolved to be a dict. Got {resolved}")
 
 
-def check_config_against_spec_or_exit(config: Mapping[str, Any], spec: ConnectorSpecification) -> None:
+def check_config_against_spec_or_exit(
+    config: Mapping[str, Any], spec: ConnectorSpecification
+) -> None:
     """
     Check config object against spec. In case of spec is invalid, throws
     an exception with validation error description.

airbyte_cdk/sources/utils/slice_logger.py

@@ -27,7 +27,10 @@ class SliceLogger(ABC):
         printable_slice = dict(_slice) if _slice else _slice
         return AirbyteMessage(
             type=MessageType.LOG,
-            log=AirbyteLogMessage(level=Level.INFO, message=f"{SliceLogger.SLICE_LOG_PREFIX}{json.dumps(printable_slice, default=str)}"),
+            log=AirbyteLogMessage(
+                level=Level.INFO,
+                message=f"{SliceLogger.SLICE_LOG_PREFIX}{json.dumps(printable_slice, default=str)}",
+            ),
         )
 
     @abstractmethod

airbyte_cdk/sources/utils/transform.py

@@ -9,7 +9,13 @@ from typing import Any, Callable, Dict, Mapping, Optional
 
 from jsonschema import Draft7Validator, ValidationError, validators
 
-json_to_python_simple = {"string": str, "number": float, "integer": int, "boolean": bool, "null": type(None)}
+json_to_python_simple = {
+    "string": str,
+    "number": float,
+    "integer": int,
+    "boolean": bool,
+    "null": type(None),
+}
 json_to_python = {**json_to_python_simple, **{"object": dict, "array": list}}
 python_to_json = {v: k for k, v in json_to_python.items()}
 
@@ -56,9 +62,13 @@ class TypeTransformer:
             # Do not validate field we do not transform for maximum performance.
             if key in ["type", "array", "$ref", "properties", "items"]
         }
-        self._normalizer = validators.create(meta_schema=Draft7Validator.META_SCHEMA, validators=all_validators)
+        self._normalizer = validators.create(
+            meta_schema=Draft7Validator.META_SCHEMA, validators=all_validators
+        )
 
-    def registerCustomTransform(self, normalization_callback: Callable[[Any, Dict[str, Any]], Any]) -> Callable:
+    def registerCustomTransform(
+        self, normalization_callback: Callable[[Any, Dict[str, Any]], Any]
+    ) -> Callable:
         """
         Register custom normalization callback.
         :param normalization_callback function to be used for value
@@ -68,7 +78,9 @@ class TypeTransformer:
         :return Same callbeck, this is usefull for using registerCustomTransform function as decorator.
         """
         if TransformConfig.CustomSchemaNormalization not in self._config:
-            raise Exception("Please set TransformConfig.CustomSchemaNormalization config before registering custom normalizer")
+            raise Exception(
+                "Please set TransformConfig.CustomSchemaNormalization config before registering custom normalizer"
+            )
         self._custom_normalizer = normalization_callback
         return normalization_callback
 
@@ -120,7 +132,10 @@ class TypeTransformer:
                 return bool(original_item)
             elif target_type == "array":
                 item_types = set(subschema.get("items", {}).get("type", set()))
-                if item_types.issubset(json_to_python_simple) and type(original_item) in json_to_python_simple.values():
+                if (
+                    item_types.issubset(json_to_python_simple)
+                    and type(original_item) in json_to_python_simple.values()
+                ):
                     return [original_item]
         except (ValueError, TypeError):
             return original_item
@@ -133,7 +148,9 @@ class TypeTransformer:
         :original_validator: native jsonschema validator callback.
         """
 
-        def normalizator(validator_instance: Callable, property_value: Any, instance: Any, schema: Dict[str, Any]):
+        def normalizator(
+            validator_instance: Callable, property_value: Any, instance: Any, schema: Dict[str, Any]
+        ):
             """
             Jsonschema validator callable it uses for validating instance. We
             override default Draft7Validator to perform value transformation
@@ -191,6 +208,4 @@ class TypeTransformer:
     def get_error_message(self, e: ValidationError) -> str:
         instance_json_type = python_to_json[type(e.instance)]
         key_path = "." + ".".join(map(str, e.path))
-        return (
-            f"Failed to transform value {repr(e.instance)} of type '{instance_json_type}' to '{e.validator_value}', key path: '{key_path}'"
-        )
+        return f"Failed to transform value {repr(e.instance)} of type '{instance_json_type}' to '{e.validator_value}', key path: '{key_path}'"

airbyte_cdk/sql/exceptions.py

@@ -90,12 +90,18 @@ class AirbyteError(Exception):
             "original_exception",
         ]
         display_properties = {
-            k: v for k, v in self.__dict__.items() if k not in special_properties and not k.startswith("_") and v is not None
+            k: v
+            for k, v in self.__dict__.items()
+            if k not in special_properties and not k.startswith("_") and v is not None
         }
         display_properties.update(self.context or {})
-        context_str = "\n    ".join(f"{str(k).replace('_', ' ').title()}: {v!r}" for k, v in display_properties.items())
+        context_str = "\n    ".join(
+            f"{str(k).replace('_', ' ').title()}: {v!r}" for k, v in display_properties.items()
+        )
         exception_str = (
-            f"{self.get_message()} ({self.__class__.__name__})" + VERTICAL_SEPARATOR + f"\n{self.__class__.__name__}: {self.get_message()}"
+            f"{self.get_message()} ({self.__class__.__name__})"
+            + VERTICAL_SEPARATOR
+            + f"\n{self.__class__.__name__}: {self.get_message()}"
         )
 
         if self.guidance:
@@ -124,7 +130,9 @@ class AirbyteError(Exception):
     def __repr__(self) -> str:
         """Return a string representation of the exception."""
         class_name = self.__class__.__name__
-        properties_str = ", ".join(f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_"))
+        properties_str = ", ".join(
+            f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_")
+        )
         return f"{class_name}({properties_str})"
 
     def safe_logging_dict(self) -> dict[str, Any]:
@@ -180,7 +188,10 @@ class AirbyteInputError(AirbyteError, ValueError):
 class AirbyteNameNormalizationError(AirbyteError, ValueError):
     """Error occurred while normalizing a table or column name."""
 
-    guidance = "Please consider renaming the source object if possible, or " "raise an issue in GitHub if not."
+    guidance = (
+        "Please consider renaming the source object if possible, or "
+        "raise an issue in GitHub if not."
+    )
     help_url = NEW_ISSUE_URL
 
     raw_name: str | None = None
@@ -205,7 +216,9 @@ class AirbyteConnectorError(AirbyteError):
             logger = logging.getLogger(f"airbyte.{self.connector_name}")
 
             log_paths: list[Path] = [
-                Path(handler.baseFilename).absolute() for handler in logger.handlers if isinstance(handler, logging.FileHandler)
+                Path(handler.baseFilename).absolute()
+                for handler in logger.handlers
+                if isinstance(handler, logging.FileHandler)
             ]
 
             if log_paths:

airbyte_cdk/sql/secrets.py

@@ -101,7 +101,9 @@ class SecretString(str):
         handler: GetCoreSchemaHandler,
     ) -> CoreSchema:
         """Return a modified core schema for the secret string."""
-        return core_schema.with_info_after_validator_function(function=cls.validate, schema=handler(str), field_name=handler.field_name)
+        return core_schema.with_info_after_validator_function(
+            function=cls.validate, schema=handler(str), field_name=handler.field_name
+        )
 
     @classmethod
     def __get_pydantic_json_schema__(  # noqa: PLW3201  # Pydantic dunder method

airbyte_cdk/sql/shared/catalog_providers.py

@@ -77,13 +77,17 @@ class CatalogProvider:
             )
 
         matching_streams: list[ConfiguredAirbyteStream] = [
-            stream for stream in self.configured_catalog.streams if stream.stream.name == stream_name
+            stream
+            for stream in self.configured_catalog.streams
+            if stream.stream.name == stream_name
         ]
         if not matching_streams:
             raise exc.AirbyteStreamNotFoundError(
                 stream_name=stream_name,
                 context={
-                    "available_streams": [stream.stream.name for stream in self.configured_catalog.streams],
+                    "available_streams": [
+                        stream.stream.name for stream in self.configured_catalog.streams
+                    ],
                 },
             )
 
@@ -121,12 +125,17 @@ class CatalogProvider:
         if not pks:
             return []
 
-        normalized_pks: list[list[str]] = [[LowerCaseNormalizer.normalize(c) for c in pk] for pk in pks]
+        normalized_pks: list[list[str]] = [
+            [LowerCaseNormalizer.normalize(c) for c in pk] for pk in pks
+        ]
 
         for pk_nodes in normalized_pks:
             if len(pk_nodes) != 1:
                 raise exc.AirbyteError(
-                    message=("Nested primary keys are not supported. " "Each PK column should have exactly one node. "),
+                    message=(
+                        "Nested primary keys are not supported. "
+                        "Each PK column should have exactly one node. "
+                    ),
                     context={
                         "stream_name": stream_name,
                         "primary_key_nodes": pk_nodes,

airbyte_cdk/sql/shared/sql_processor.py

@@ -16,7 +16,12 @@ import ulid
 from airbyte_cdk.sql import exceptions as exc
 from airbyte_cdk.sql._util.hashing import one_way_hash
 from airbyte_cdk.sql._util.name_normalizers import LowerCaseNormalizer
-from airbyte_cdk.sql.constants import AB_EXTRACTED_AT_COLUMN, AB_META_COLUMN, AB_RAW_ID_COLUMN, DEBUG_MODE
+from airbyte_cdk.sql.constants import (
+    AB_EXTRACTED_AT_COLUMN,
+    AB_META_COLUMN,
+    AB_RAW_ID_COLUMN,
+    DEBUG_MODE,
+)
 from airbyte_cdk.sql.secrets import SecretString
 from airbyte_cdk.sql.types import SQLTypeConverter
 from airbyte_protocol_dataclasses.models import AirbyteStateMessage
@@ -100,7 +105,9 @@ class SqlConfig(BaseModel, abc.ABC):
 
         Raises `NotImplementedError` if a custom vendor client is not defined.
         """
-        raise NotImplementedError(f"The type '{type(self).__name__}' does not define a custom client.")
+        raise NotImplementedError(
+            f"The type '{type(self).__name__}' does not define a custom client."
+        )
 
 
 class SqlProcessorBase(abc.ABC):
@@ -270,7 +277,9 @@ class SqlProcessorBase(abc.ABC):
         query. To ignore the cache and force a refresh, set 'force_refresh' to True.
         """
         if force_refresh and shallow_okay:
-            raise exc.AirbyteInternalError(message="Cannot force refresh and use shallow query at the same time.")
+            raise exc.AirbyteInternalError(
+                message="Cannot force refresh and use shallow query at the same time."
+            )
 
         if force_refresh and table_name in self._cached_table_definitions:
             self._invalidate_table_cache(table_name)
@@ -315,7 +324,9 @@ class SqlProcessorBase(abc.ABC):
 
         if DEBUG_MODE:
             found_schemas = schemas_list
-            assert schema_name in found_schemas, f"Schema {schema_name} was not created. Found: {found_schemas}"
+            assert (
+                schema_name in found_schemas
+            ), f"Schema {schema_name} was not created. Found: {found_schemas}"
 
     def _quote_identifier(self, identifier: str) -> str:
         """Return the given identifier, quoted."""
@@ -387,7 +398,8 @@ class SqlProcessorBase(abc.ABC):
         self._known_schemas_list = [
             found_schema.split(".")[-1].strip('"')
             for found_schema in found_schemas
-            if "." not in found_schema or (found_schema.split(".")[0].lower().strip('"') == database_name.lower())
+            if "." not in found_schema
+            or (found_schema.split(".")[0].lower().strip('"') == database_name.lower())
         ]
         return self._known_schemas_list
 
@@ -511,7 +523,9 @@ class SqlProcessorBase(abc.ABC):
         for file_path in files:
             dataframe = pd.read_json(file_path, lines=True)
 
-            sql_column_definitions: dict[str, TypeEngine[Any]] = self._get_sql_column_definitions(stream_name)
+            sql_column_definitions: dict[str, TypeEngine[Any]] = self._get_sql_column_definitions(
+                stream_name
+            )
 
             # Remove fields that are not in the schema
             for col_name in dataframe.columns:
@@ -549,7 +563,10 @@ class SqlProcessorBase(abc.ABC):
     ) -> None:
         """Add a column to the given table."""
         self._execute_sql(
-            text(f"ALTER TABLE {self._fully_qualified(table.name)} " f"ADD COLUMN {column_name} {column_type}"),
+            text(
+                f"ALTER TABLE {self._fully_qualified(table.name)} "
+                f"ADD COLUMN {column_name} {column_type}"
+            ),
         )
 
     def _add_missing_columns_to_table(
@@ -626,8 +643,10 @@ class SqlProcessorBase(abc.ABC):
         deletion_name = f"{final_table_name}_deleteme"
         commands = "\n".join(
             [
-                f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME " f"TO {deletion_name};",
-                f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME " f"TO {final_table_name};",
+                f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME "
+                f"TO {deletion_name};",
+                f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME "
+                f"TO {final_table_name};",
                 f"DROP TABLE {self._fully_qualified(deletion_name)};",
             ]
         )
@@ -646,7 +665,9 @@ class SqlProcessorBase(abc.ABC):
         """
         nl = "\n"
         columns = {self._quote_identifier(c) for c in self._get_sql_column_definitions(stream_name)}
-        pk_columns = {self._quote_identifier(c) for c in self.catalog_provider.get_primary_keys(stream_name)}
+        pk_columns = {
+            self._quote_identifier(c) for c in self.catalog_provider.get_primary_keys(stream_name)
+        }
         non_pk_columns = columns - pk_columns
         join_clause = f"{nl} AND ".join(f"tmp.{pk_col} = final.{pk_col}" for pk_col in pk_columns)
         set_clause = f"{nl}  , ".join(f"{col} = tmp.{col}" for col in non_pk_columns)
@@ -704,16 +725,23 @@ class SqlProcessorBase(abc.ABC):
         temp_table = self._get_table_by_name(temp_table_name)
         pk_columns = self.catalog_provider.get_primary_keys(stream_name)
 
-        columns_to_update: set[str] = self._get_sql_column_definitions(stream_name=stream_name).keys() - set(pk_columns)
+        columns_to_update: set[str] = self._get_sql_column_definitions(
+            stream_name=stream_name
+        ).keys() - set(pk_columns)
 
         # Create a dictionary mapping columns in users_final to users_stage for updating
         update_values = {
-            self._get_column_by_name(final_table, column): (self._get_column_by_name(temp_table, column)) for column in columns_to_update
+            self._get_column_by_name(final_table, column): (
+                self._get_column_by_name(temp_table, column)
+            )
+            for column in columns_to_update
         }
 
         # Craft the WHERE clause for composite primary keys
         join_conditions = [
-            self._get_column_by_name(final_table, pk_column) == self._get_column_by_name(temp_table, pk_column) for pk_column in pk_columns
+            self._get_column_by_name(final_table, pk_column)
+            == self._get_column_by_name(temp_table, pk_column)
+            for pk_column in pk_columns
         ]
         join_clause = and_(*join_conditions)
 
@@ -728,7 +756,9 @@ class SqlProcessorBase(abc.ABC):
         where_not_exists_clause = self._get_column_by_name(final_table, pk_columns[0]) == null()
 
         # Select records from temp_table that are not in final_table
-        select_new_records_stmt = select(temp_table).select_from(joined_table).where(where_not_exists_clause)
+        select_new_records_stmt = (
+            select(temp_table).select_from(joined_table).where(where_not_exists_clause)
+        )
 
         # Craft the INSERT statement using the select statement
         insert_new_records_stmt = insert(final_table).from_select(