airbyte-cdk 6.31.2.dev0__py3-none-any.whl → 6.33.0__py3-none-any.whl

airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py

@@ -2,10 +2,10 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+from datetime import timedelta
 from typing import Any, List, Mapping, Optional, Sequence, Tuple, Union
 
 import dpath
-import pendulum
 
 from airbyte_cdk.config_observation import (
     create_connector_config_control_message,
@@ -15,6 +15,11 @@ from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import (
     AbstractOauth2Authenticator,
 )
+from airbyte_cdk.utils.datetime_helpers import (
+    AirbyteDateTime,
+    ab_datetime_now,
+    ab_datetime_parse,
+)
 
 
 class Oauth2Authenticator(AbstractOauth2Authenticator):
@@ -34,7 +39,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
         client_secret_name: str = "client_secret",
         refresh_token_name: str = "refresh_token",
         scopes: List[str] | None = None,
-        token_expiry_date: pendulum.DateTime | None = None,
+        token_expiry_date: AirbyteDateTime | None = None,
         token_expiry_date_format: str | None = None,
         access_token_name: str = "access_token",
         expires_in_name: str = "expires_in",
@@ -46,7 +51,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
         refresh_token_error_status_codes: Tuple[int, ...] = (),
         refresh_token_error_key: str = "",
         refresh_token_error_values: Tuple[str, ...] = (),
-    ):
+    ) -> None:
         self._token_refresh_endpoint = token_refresh_endpoint
         self._client_secret_name = client_secret_name
         self._client_secret = client_secret
@@ -62,7 +67,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
         self._grant_type_name = grant_type_name
         self._grant_type = grant_type
 
-        self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1)  # type: ignore [no-untyped-call]
+        self._token_expiry_date = token_expiry_date or (ab_datetime_now() - timedelta(days=1))
         self._token_expiry_date_format = token_expiry_date_format
         self._token_expiry_is_time_of_expiration = token_expiry_is_time_of_expiration
         self._access_token = None
@@ -95,16 +100,16 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
         return self._access_token_name
 
     def get_scopes(self) -> list[str]:
-        return self._scopes  # type: ignore [return-value]
+        return self._scopes  # type: ignore[return-value]
 
     def get_expires_in_name(self) -> str:
         return self._expires_in_name
 
     def get_refresh_request_body(self) -> Mapping[str, Any]:
-        return self._refresh_request_body  # type: ignore [return-value]
+        return self._refresh_request_body  # type: ignore[return-value]
 
     def get_refresh_request_headers(self) -> Mapping[str, Any]:
-        return self._refresh_request_headers  # type: ignore [return-value]
+        return self._refresh_request_headers  # type: ignore[return-value]
 
     def get_grant_type_name(self) -> str:
         return self._grant_type_name
@@ -112,7 +117,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
     def get_grant_type(self) -> str:
         return self._grant_type
 
-    def get_token_expiry_date(self) -> pendulum.DateTime:
+    def get_token_expiry_date(self) -> AirbyteDateTime:
         return self._token_expiry_date
 
     def set_token_expiry_date(self, value: Union[str, int]) -> None:
@@ -128,11 +133,11 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
 
     @property
     def access_token(self) -> str:
-        return self._access_token  # type: ignore [return-value]
+        return self._access_token  # type: ignore[return-value]
 
     @access_token.setter
     def access_token(self, value: str) -> None:
-        self._access_token = value  # type: ignore [assignment]  # Incorrect type for assignment
+        self._access_token = value  # type: ignore[assignment]  # Incorrect type for assignment
 
 
 class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
@@ -170,7 +175,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         refresh_token_error_status_codes: Tuple[int, ...] = (),
         refresh_token_error_key: str = "",
         refresh_token_error_values: Tuple[str, ...] = (),
-    ):
+    ) -> None:
         """
         Args:
             connector_config (Mapping[str, Any]): The full connector configuration
@@ -191,18 +196,12 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             token_expiry_is_time_of_expiration bool: set True it if expires_in is returned as time of expiration instead of the number seconds until expiration
             message_repository (MessageRepository): the message repository used to emit logs on HTTP requests and control message on config update
         """
-        self._client_id = (
-            client_id  # type: ignore [assignment]  # Incorrect type for assignment
-            if client_id is not None
-            else dpath.get(connector_config, ("credentials", "client_id"))  # type: ignore [arg-type]
+        self._connector_config = connector_config
+        self._client_id: str = self._get_config_value_by_path(
+            ("credentials", "client_id"), client_id
         )
-        self._client_secret = (
-            client_secret  # type: ignore [assignment]  # Incorrect type for assignment
-            if client_secret is not None
-            else dpath.get(
-                connector_config,  # type: ignore [arg-type]
-                ("credentials", "client_secret"),
-            )
+        self._client_secret: str = self._get_config_value_by_path(
+            ("credentials", "client_secret"), client_secret
         )
         self._client_id_name = client_id_name
         self._client_secret_name = client_secret_name
@@ -217,9 +216,9 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
         super().__init__(
             token_refresh_endpoint=token_refresh_endpoint,
             client_id_name=self._client_id_name,
-            client_id=self.get_client_id(),
+            client_id=self._client_id,
             client_secret_name=self._client_secret_name,
-            client_secret=self.get_client_secret(),
+            client_secret=self._client_secret,
             refresh_token=self.get_refresh_token(),
             refresh_token_name=self._refresh_token_name,
             scopes=scopes,
@@ -237,76 +236,105 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             refresh_token_error_values=refresh_token_error_values,
         )
 
-    def get_refresh_token_name(self) -> str:
-        return self._refresh_token_name
-
-    def get_client_id(self) -> str:
-        return self._client_id
-
-    def get_client_secret(self) -> str:
-        return self._client_secret
-
     @property
     def access_token(self) -> str:
-        return dpath.get(  # type: ignore [return-value]
-            self._connector_config,  # type: ignore [arg-type]
-            self._access_token_config_path,
-            default="",
-        )
+        """
+        Retrieve the access token from the configuration.
+
+        Returns:
+            str: The access token.
+        """
+        return self._get_config_value_by_path(self._access_token_config_path)  # type: ignore[return-value]
 
     @access_token.setter
     def access_token(self, new_access_token: str) -> None:
-        dpath.new(
-            self._connector_config,  # type: ignore [arg-type]
-            self._access_token_config_path,
-            new_access_token,
-        )
+        """
+        Sets a new access token.
+
+        Args:
+            new_access_token (str): The new access token to be set.
+        """
+        self._set_config_value_by_path(self._access_token_config_path, new_access_token)
 
     def get_refresh_token(self) -> str:
-        return dpath.get(  # type: ignore [return-value]
-            self._connector_config,  # type: ignore [arg-type]
-            self._refresh_token_config_path,
-            default="",
-        )
+        """
+        Retrieve the refresh token from the configuration.
+
+        This method fetches the refresh token using the configuration path specified
+        by `_refresh_token_config_path`.
+
+        Returns:
+            str: The refresh token as a string.
+        """
+        return self._get_config_value_by_path(self._refresh_token_config_path)  # type: ignore[return-value]
 
     def set_refresh_token(self, new_refresh_token: str) -> None:
-        dpath.new(
-            self._connector_config,  # type: ignore [arg-type]
-            self._refresh_token_config_path,
-            new_refresh_token,
-        )
+        """
+        Updates the refresh token in the configuration.
+
+        Args:
+            new_refresh_token (str): The new refresh token to be set.
+        """
+        self._set_config_value_by_path(self._refresh_token_config_path, new_refresh_token)
+
+    def get_token_expiry_date(self) -> AirbyteDateTime:
+        """
+        Retrieves the token expiry date from the configuration.
+
+        This method fetches the token expiry date from the configuration using the specified path.
+        If the expiry date is an empty string, it returns the current date and time minus one day.
+        Otherwise, it parses the expiry date string into an AirbyteDateTime object.
 
-    def get_token_expiry_date(self) -> pendulum.DateTime:
-        expiry_date = dpath.get(
-            self._connector_config,  # type: ignore [arg-type]
-            self._token_expiry_date_config_path,
-            default="",
+        Returns:
+            AirbyteDateTime: The parsed or calculated token expiry date.
+
+        Raises:
+            TypeError: If the result is not an instance of AirbyteDateTime.
+        """
+        expiry_date = self._get_config_value_by_path(self._token_expiry_date_config_path)
+        result = (
+            ab_datetime_now() - timedelta(days=1)
+            if expiry_date == ""
+            else ab_datetime_parse(str(expiry_date))
         )
-        return pendulum.now().subtract(days=1) if expiry_date == "" else pendulum.parse(expiry_date)  # type: ignore [arg-type, return-value, no-untyped-call]
+        if isinstance(result, AirbyteDateTime):
+            return result
+        raise TypeError("Invalid datetime conversion")
 
-    def set_token_expiry_date(  # type: ignore[override]
-        self,
-        new_token_expiry_date: pendulum.DateTime,
-    ) -> None:
-        dpath.new(
-            self._connector_config,  # type: ignore [arg-type]
-            self._token_expiry_date_config_path,
-            str(new_token_expiry_date),
+    def set_token_expiry_date(self, new_token_expiry_date: AirbyteDateTime) -> None:  # type: ignore[override]
+        """
+        Sets the token expiry date in the configuration.
+
+        Args:
+            new_token_expiry_date (AirbyteDateTime): The new expiry date for the token.
+        """
+        self._set_config_value_by_path(
+            self._token_expiry_date_config_path, str(new_token_expiry_date)
         )
 
     def token_has_expired(self) -> bool:
         """Returns True if the token is expired"""
-        return pendulum.now("UTC") > self.get_token_expiry_date()
+        return ab_datetime_now() > self.get_token_expiry_date()
 
     @staticmethod
     def get_new_token_expiry_date(
         access_token_expires_in: str,
         token_expiry_date_format: str | None = None,
-    ) -> pendulum.DateTime:
+    ) -> AirbyteDateTime:
+        """
+        Calculate the new token expiry date based on the provided expiration duration or format.
+
+        Args:
+            access_token_expires_in (str): The duration (in seconds) until the access token expires, or the expiry date in a specific format.
+            token_expiry_date_format (str | None, optional): The format of the expiry date if provided. Defaults to None.
+
+        Returns:
+            AirbyteDateTime: The calculated expiry date of the access token.
+        """
         if token_expiry_date_format:
-            return pendulum.from_format(access_token_expires_in, token_expiry_date_format)
+            return ab_datetime_parse(access_token_expires_in)
         else:
-            return pendulum.now("UTC").add(seconds=int(access_token_expires_in))
+            return ab_datetime_now() + timedelta(seconds=int(access_token_expires_in))
 
     def get_access_token(self) -> str:
         """Retrieve new access and refresh token if the access token has expired.
@@ -318,33 +346,88 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
             new_access_token, access_token_expires_in, new_refresh_token = (
                 self.refresh_access_token()
             )
-            new_token_expiry_date: pendulum.DateTime = self.get_new_token_expiry_date(
+            new_token_expiry_date: AirbyteDateTime = self.get_new_token_expiry_date(
                 access_token_expires_in, self._token_expiry_date_format
             )
             self.access_token = new_access_token
             self.set_refresh_token(new_refresh_token)
             self.set_token_expiry_date(new_token_expiry_date)
-            # FIXME emit_configuration_as_airbyte_control_message as been deprecated in favor of package airbyte_cdk.sources.message
-            #  Usually, a class shouldn't care about the implementation details but to keep backward compatibility where we print the
-            #  message directly in the console, this is needed
-            if not isinstance(self._message_repository, NoopMessageRepository):
-                self._message_repository.emit_message(
-                    create_connector_config_control_message(self._connector_config)  # type: ignore [arg-type]
-                )
-            else:
-                emit_configuration_as_airbyte_control_message(self._connector_config)  # type: ignore [arg-type]
+            self._emit_control_message()
         return self.access_token
 
-    def refresh_access_token(  # type: ignore[override]  # Signature doesn't match base class
-        self,
-    ) -> Tuple[str, str, str]:
-        response_json = self._get_refresh_access_token_response()
+    def refresh_access_token(self) -> Tuple[str, str, str]:  # type: ignore[override]
+        """
+        Refreshes the access token by making a handled request and extracting the necessary token information.
+
+        Returns:
+            Tuple[str, str, str]: A tuple containing the new access token, token expiry date, and refresh token.
+        """
+        response_json = self._make_handled_request()
         return (
-            response_json[self.get_access_token_name()],
-            response_json[self.get_expires_in_name()],
-            response_json[self.get_refresh_token_name()],
+            self._extract_access_token(response_json),
+            self._extract_token_expiry_date(response_json),
+            self._extract_refresh_token(response_json),
+        )
+
+    def _set_config_value_by_path(self, config_path: Union[str, Sequence[str]], value: Any) -> None:
+        """
+        Set a value in the connector configuration at the specified path.
+
+        Args:
+            config_path (Union[str, Sequence[str]]): The path within the configuration where the value should be set.
+                This can be a string representing a single key or a sequence of strings representing a nested path.
+            value (Any): The value to set at the specified path in the configuration.
+
+        Returns:
+            None
+        """
+        dpath.new(self._connector_config, config_path, value)  # type: ignore[arg-type]
+
+    def _get_config_value_by_path(
+        self, config_path: Union[str, Sequence[str]], default: Optional[str] = None
+    ) -> str | Any:
+        """
+        Retrieve a value from the connector configuration using a specified path.
+
+        Args:
+            config_path (Union[str, Sequence[str]]): The path to the desired configuration value. This can be a string or a sequence of strings.
+            default (Optional[str], optional): The default value to return if the specified path does not exist in the configuration. Defaults to None.
+
+        Returns:
+            Any: The value from the configuration at the specified path, or the default value if the path does not exist.
+        """
+        return dpath.get(
+            self._connector_config,  # type: ignore[arg-type]
+            config_path,
+            default=default if default is not None else "",
         )
 
+    def _emit_control_message(self) -> None:
+        """
+        Emits a control message based on the connector configuration.
+
+        This method checks if the message repository is not a NoopMessageRepository.
+        If it is not, it emits a message using the message repository. Otherwise,
+        it falls back to emitting the configuration as an Airbyte control message
+        directly to the console for backward compatibility.
+
+        Note:
+            The function `emit_configuration_as_airbyte_control_message` has been deprecated
+            in favor of the package `airbyte_cdk.sources.message`.
+
+        Raises:
+            TypeError: If the argument types are incorrect.
+        """
+        # FIXME emit_configuration_as_airbyte_control_message as been deprecated in favor of package airbyte_cdk.sources.message
+        # Usually, a class shouldn't care about the implementation details but to keep backward compatibility where we print the
+        # message directly in the console, this is needed
+        if not isinstance(self._message_repository, NoopMessageRepository):
+            self._message_repository.emit_message(
+                create_connector_config_control_message(self._connector_config)  # type: ignore[arg-type]
+            )
+        else:
+            emit_configuration_as_airbyte_control_message(self._connector_config)  # type: ignore[arg-type]
+
     @property
     def _message_repository(self) -> MessageRepository:
         """

airbyte_cdk/sources/types.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 
 from typing import Any, ItemsView, Iterator, KeysView, List, Mapping, Optional, ValuesView
 
-import orjson
+from airbyte_cdk.utils.slice_hasher import SliceHasher
 
 # A FieldPointer designates a path to a field inside a mapping. For example, retrieving ["k1", "k1.2"] in the object {"k1" :{"k1.2":
 # "hello"}] returns "hello"
@@ -151,7 +151,9 @@ class StreamSlice(Mapping[str, Any]):
         return self._stream_slice
 
     def __hash__(self) -> int:
-        return hash(orjson.dumps(self._stream_slice, option=orjson.OPT_SORT_KEYS))
+        return SliceHasher.hash(
+            stream_slice=self._stream_slice
+        )  # no need to provide stream_name here as this is used for slicing the cursor
 
     def __bool__(self) -> bool:
         return bool(self._stream_slice) or bool(self._extra_fields)

airbyte_cdk/sources/utils/transform.py

@@ -3,7 +3,6 @@
 #
 
 import logging
-from distutils.util import strtobool
 from enum import Flag, auto
 from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
 
@@ -22,6 +21,28 @@ python_to_json = {v: k for k, v in json_to_python.items()}
 
 logger = logging.getLogger("airbyte")
 
+_TRUTHY_STRINGS = ("y", "yes", "t", "true", "on", "1")
+_FALSEY_STRINGS = ("n", "no", "f", "false", "off", "0")
+
+
+def _strtobool(value: str, /) -> int:
+    """Mimic the behavior of distutils.util.strtobool.
+
+    From: https://docs.python.org/2/distutils/apiref.html#distutils.util.strtobool
+
+    > Convert a string representation of truth to true (1) or false (0).
+    > True values are y, yes, t, true, on and 1; false values are n, no, f, false, off and 0. Raises
+    > `ValueError` if val is anything else.
+    """
+    normalized_str = value.lower().strip()
+    if normalized_str in _TRUTHY_STRINGS:
+        return 1
+
+    if normalized_str in _FALSEY_STRINGS:
+        return 0
+
+    raise ValueError(f"Invalid boolean value: {normalized_str}")
+
 
 class TransformConfig(Flag):
     """
@@ -129,7 +150,7 @@ class TypeTransformer:
                 return int(original_item)
             elif target_type == "boolean":
                 if isinstance(original_item, str):
-                    return strtobool(original_item) == 1
+                    return _strtobool(original_item) == 1
                 return bool(original_item)
             elif target_type == "array":
                 item_types = set(subschema.get("items", {}).get("type", set()))

airbyte_cdk/test/utils/manifest_only_fixtures.py

@@ -4,7 +4,6 @@
 import importlib.util
 from pathlib import Path
 from types import ModuleType
-from typing import Optional
 
 import pytest
 
@@ -30,7 +29,7 @@ def connector_dir(request: pytest.FixtureRequest) -> Path:
 
 
 @pytest.fixture(scope="session")
-def components_module(connector_dir: Path) -> Optional[ModuleType]:
+def components_module(connector_dir: Path) -> ModuleType | None:
     """Load and return the components module from the connector directory.
 
     This assumes the components module is located at <connector_dir>/components.py.