airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/auth/token.py

@@ -0,0 +1,267 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import base64
+import logging
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, MutableMapping, Union
+
+import requests
+from cachetools import TTLCache, cached
+
+from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
+from airbyte_cdk.sources.declarative.auth.token_provider import TokenProvider
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.requesters.request_option import (
+    RequestOption,
+    RequestOptionType,
+)
+from airbyte_cdk.sources.types import Config
+
+
+@dataclass
+class ApiKeyAuthenticator(DeclarativeAuthenticator):
+    """
+    ApiKeyAuth sets a request header on the HTTP requests sent.
+
+    The header is of the form:
+    `"<header>": "<token>"`
+
+    For example,
+    `ApiKeyAuthenticator("Authorization", "Bearer hello")`
+    will result in the following header set on the HTTP request
+    `"Authorization": "Bearer hello"`
+
+    Attributes:
+        request_option (RequestOption): request option how to inject the token into the request
+        token_provider (TokenProvider): Provider of the token
+        config (Config): The user-provided configuration as specified by the source's spec
+        parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation
+    """
+
+    request_option: RequestOption
+    token_provider: TokenProvider
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+
+    @property
+    def auth_header(self) -> str:
+        options = self._get_request_options(RequestOptionType.header)
+        return next(iter(options.keys()), "")
+
+    @property
+    def token(self) -> str:
+        return self.token_provider.get_token()
+
+    def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, Any]:
+        options: MutableMapping[str, Any] = {}
+        if self.request_option.inject_into == option_type:
+            self.request_option.inject_into_request(options, self.token, self.config)
+        return options
+
+    def get_request_params(self) -> Mapping[str, Any]:
+        return self._get_request_options(RequestOptionType.request_parameter)
+
+    def get_request_body_data(self) -> Union[Mapping[str, Any], str]:
+        return self._get_request_options(RequestOptionType.body_data)
+
+    def get_request_body_json(self) -> Mapping[str, Any]:
+        return self._get_request_options(RequestOptionType.body_json)
+
+
+@dataclass
+class BearerAuthenticator(DeclarativeAuthenticator):
+    """
+    Authenticator that sets the Authorization header on the HTTP requests sent.
+
+    The header is of the form:
+    `"Authorization": "Bearer <token>"`
+
+    Attributes:
+        token_provider (TokenProvider): Provider of the token
+        config (Config): The user-provided configuration as specified by the source's spec
+        parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation
+    """
+
+    token_provider: TokenProvider
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+
+    @property
+    def auth_header(self) -> str:
+        return "Authorization"
+
+    @property
+    def token(self) -> str:
+        return f"Bearer {self.token_provider.get_token()}"
+
+
+@dataclass
+class BasicHttpAuthenticator(DeclarativeAuthenticator):
+    """
+    Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using base64
+    https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme
+
+    The header is of the form
+    `"Authorization": "Basic <encoded_credentials>"`
+
+    Attributes:
+        username (Union[InterpolatedString, str]): The username
+        config (Config): The user-provided configuration as specified by the source's spec
+        password (Union[InterpolatedString, str]): The password
+        parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation
+    """
+
+    username: Union[InterpolatedString, str]
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+    password: Union[InterpolatedString, str] = ""
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._username = InterpolatedString.create(self.username, parameters=parameters)
+        self._password = InterpolatedString.create(self.password, parameters=parameters)
+
+    @property
+    def auth_header(self) -> str:
+        return "Authorization"
+
+    @property
+    def token(self) -> str:
+        auth_string = (
+            f"{self._username.eval(self.config)}:{self._password.eval(self.config)}".encode("utf8")
+        )
+        b64_encoded = base64.b64encode(auth_string).decode("utf8")
+        return f"Basic {b64_encoded}"
+
+
+"""
+    maxsize - The maximum size of the cache
+    ttl - time-to-live value in seconds
+    docs https://cachetools.readthedocs.io/en/latest/
+    maxsize=1000 - when the cache is full, in this case more than 1000,
+    i.e. by adding another item the cache would exceed its maximum size, the cache must choose which item(s) to discard
+    ttl=86400 means that cached token will live for 86400 seconds (one day)
+"""
+cacheSessionTokenAuthenticator: TTLCache[str, str] = TTLCache(maxsize=1000, ttl=86400)
+
+
+@cached(cacheSessionTokenAuthenticator)
+def get_new_session_token(api_url: str, username: str, password: str, response_key: str) -> str:
+    """
+    This method retrieves session token from api by username and password for SessionTokenAuthenticator.
+    It's cashed to avoid a multiple calling by sync and updating session token every stream sync.
+    Args:
+        api_url: api url for getting new session token
+        username: username for auth
+        password: password for auth
+        response_key: field name in response to retrieve a session token
+
+    Returns:
+        session token
+    """
+    response = requests.post(
+        f"{api_url}",
+        headers={"Content-Type": "application/json"},
+        json={"username": username, "password": password},
+    )
+    response.raise_for_status()
+    if not response.ok:
+        raise ConnectionError(
+            f"Failed to retrieve new session token, response code {response.status_code} because {response.reason}"
+        )
+    return str(response.json()[response_key])
+
+
+@dataclass
+class LegacySessionTokenAuthenticator(DeclarativeAuthenticator):
+    """
+    Builds auth based on session tokens.
+    A session token is a random value generated by a server to identify
+    a specific user for the duration of one interaction session.
+
+    The header is of the form
+    `"Specific Header": "Session Token Value"`
+
+    Attributes:
+        api_url (Union[InterpolatedString, str]): Base api url of source
+        username (Union[InterpolatedString, str]): The username
+        config (Config): The user-provided configuration as specified by the source's spec
+        password (Union[InterpolatedString, str]): The password
+        header (Union[InterpolatedString, str]): Specific header of source for providing session token
+        parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation
+        session_token (Union[InterpolatedString, str]): Session token generated by user
+        session_token_response_key (Union[InterpolatedString, str]): Key for retrieving session token from api response
+        login_url (Union[InterpolatedString, str]): Url fot getting a specific session token
+        validate_session_url (Union[InterpolatedString, str]): Url to validate passed session token
+    """
+
+    api_url: Union[InterpolatedString, str]
+    header: Union[InterpolatedString, str]
+    session_token: Union[InterpolatedString, str]
+    session_token_response_key: Union[InterpolatedString, str]
+    username: Union[InterpolatedString, str]
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+    login_url: Union[InterpolatedString, str]
+    validate_session_url: Union[InterpolatedString, str]
+    password: Union[InterpolatedString, str] = ""
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._username = InterpolatedString.create(self.username, parameters=parameters)
+        self._password = InterpolatedString.create(self.password, parameters=parameters)
+        self._api_url = InterpolatedString.create(self.api_url, parameters=parameters)
+        self._header = InterpolatedString.create(self.header, parameters=parameters)
+        self._session_token = InterpolatedString.create(self.session_token, parameters=parameters)
+        self._session_token_response_key = InterpolatedString.create(
+            self.session_token_response_key, parameters=parameters
+        )
+        self._login_url = InterpolatedString.create(self.login_url, parameters=parameters)
+        self._validate_session_url = InterpolatedString.create(
+            self.validate_session_url, parameters=parameters
+        )
+
+        self.logger = logging.getLogger("airbyte")
+
+    @property
+    def auth_header(self) -> str:
+        return str(self._header.eval(self.config))
+
+    @property
+    def token(self) -> str:
+        if self._session_token.eval(self.config):
+            if self.is_valid_session_token():
+                return str(self._session_token.eval(self.config))
+        if self._password.eval(self.config) and self._username.eval(self.config):
+            username = self._username.eval(self.config)
+            password = self._password.eval(self.config)
+            session_token_response_key = self._session_token_response_key.eval(self.config)
+            api_url = f"{self._api_url.eval(self.config)}{self._login_url.eval(self.config)}"
+
+            self.logger.info("Using generated session token by username and password")
+            return get_new_session_token(api_url, username, password, session_token_response_key)
+
+        raise ConnectionError(
+            "Invalid credentials: session token is not valid or provide username and password"
+        )
+
+    def is_valid_session_token(self) -> bool:
+        try:
+            response = requests.get(
+                f"{self._api_url.eval(self.config)}{self._validate_session_url.eval(self.config)}",
+                headers={self.auth_header: self._session_token.eval(self.config)},
+            )
+            response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == requests.codes["unauthorized"]:
+                self.logger.info(f"Unable to connect by session token from config due to {str(e)}")
+                return False
+            else:
+                raise ConnectionError(f"Error while validating session token: {e}")
+        if response.ok:
+            self.logger.info("Connection check for source is successful.")
+            return True
+        else:
+            raise ConnectionError(
+                f"Failed to retrieve new session token, response code {response.status_code} because {response.reason}"
+            )

airbyte_cdk/sources/declarative/auth/token_provider.py

@@ -0,0 +1,82 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import datetime
+from abc import abstractmethod
+from dataclasses import InitVar, dataclass, field
+from typing import Any, List, Mapping, Optional, Union
+
+import dpath
+from isodate import Duration
+
+from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
+from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
+from airbyte_cdk.sources.declarative.exceptions import ReadException
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.requesters.requester import Requester
+from airbyte_cdk.sources.http_logger import format_http_message
+from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
+from airbyte_cdk.sources.types import Config
+from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now
+
+
+class TokenProvider:
+    @abstractmethod
+    def get_token(self) -> str:
+        pass
+
+
+@dataclass
+class SessionTokenProvider(TokenProvider):
+    login_requester: Requester
+    session_token_path: List[str]
+    expiration_duration: Optional[Union[datetime.timedelta, Duration]]
+    parameters: InitVar[Mapping[str, Any]]
+    message_repository: MessageRepository = NoopMessageRepository()
+    decoder: Decoder = field(default_factory=lambda: JsonDecoder(parameters={}))
+
+    _next_expiration_time: Optional[AirbyteDateTime] = None
+    _token: Optional[str] = None
+
+    def get_token(self) -> str:
+        self._refresh_if_necessary()
+        if self._token is None:
+            raise ReadException("Failed to get session token, token is None")
+        return self._token
+
+    def _refresh_if_necessary(self) -> None:
+        if self._next_expiration_time is None or self._next_expiration_time < ab_datetime_now():
+            self._refresh()
+
+    def _refresh(self) -> None:
+        response = self.login_requester.send_request(
+            log_formatter=lambda response: format_http_message(
+                response,
+                "Login request",
+                "Obtains session token",
+                None,
+                is_auxiliary=True,
+                type="AUTH",
+            ),
+        )
+        if response is None:
+            raise ReadException("Failed to get session token, response got ignored by requester")
+        session_token = dpath.get(next(self.decoder.decode(response)), self.session_token_path)
+        if self.expiration_duration is not None:
+            self._next_expiration_time = ab_datetime_now() + self.expiration_duration
+        self._token = session_token  # type: ignore # Returned decoded response will be Mapping and therefore session_token will be str or None
+
+
+@dataclass
+class InterpolatedStringTokenProvider(TokenProvider):
+    config: Config
+    api_token: Union[InterpolatedString, str]
+    parameters: Mapping[str, Any]
+
+    def __post_init__(self) -> None:
+        self._token = InterpolatedString.create(self.api_token, parameters=self.parameters)
+
+    def get_token(self) -> str:
+        return str(self._token.eval(self.config))

airbyte_cdk/sources/declarative/checks/__init__.py

@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Mapping
+
+from pydantic.v1 import BaseModel
+
+from airbyte_cdk.sources.declarative.checks.check_dynamic_stream import CheckDynamicStream
+from airbyte_cdk.sources.declarative.checks.check_stream import CheckStream
+from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
+from airbyte_cdk.sources.declarative.models import (
+    CheckDynamicStream as CheckDynamicStreamModel,
+)
+from airbyte_cdk.sources.declarative.models import (
+    CheckStream as CheckStreamModel,
+)
+
+COMPONENTS_CHECKER_TYPE_MAPPING: Mapping[str, type[BaseModel]] = {
+    "CheckStream": CheckStreamModel,
+    "CheckDynamicStream": CheckDynamicStreamModel,
+}
+
+__all__ = ["CheckStream", "CheckDynamicStream", "ConnectionChecker"]

airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+import traceback
+from dataclasses import InitVar, dataclass
+from typing import Any, List, Mapping, Tuple
+
+from airbyte_cdk import AbstractSource
+from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
+from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
+
+
+@dataclass
+class CheckDynamicStream(ConnectionChecker):
+    """
+    Checks the connections by checking availability of one or many dynamic streams
+
+    Attributes:
+        stream_count (int): numbers of streams to check
+    """
+
+    # TODO: Add field stream_names to check_connection for static streams
+    #  https://github.com/airbytehq/airbyte-python-cdk/pull/293#discussion_r1934933483
+
+    stream_count: int
+    parameters: InitVar[Mapping[str, Any]]
+    use_check_availability: bool = True
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._parameters = parameters
+
+    def check_connection(
+        self, source: AbstractSource, logger: logging.Logger, config: Mapping[str, Any]
+    ) -> Tuple[bool, Any]:
+        streams = source.streams(config=config)
+
+        if len(streams) == 0:
+            return False, f"No streams to connect to from source {source}"
+        if not self.use_check_availability:
+            return True, None
+
+        availability_strategy = HttpAvailabilityStrategy()
+
+        try:
+            for stream in streams[: min(self.stream_count, len(streams))]:
+                stream_is_available, reason = availability_strategy.check_availability(
+                    stream, logger
+                )
+                if not stream_is_available:
+                    logger.warning(f"Stream {stream.name} is not available: {reason}")
+                    return False, reason
+        except Exception as error:
+            error_message = (
+                f"Encountered an error trying to connect to stream {stream.name}. Error: {error}"
+            )
+            logger.error(error_message, exc_info=True)
+            return False, error_message
+
+        return True, None

airbyte_cdk/sources/declarative/checks/check_stream.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+import traceback
+from dataclasses import InitVar, dataclass
+from typing import Any, List, Mapping, Tuple
+
+from airbyte_cdk import AbstractSource
+from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
+from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
+
+
+@dataclass
+class CheckStream(ConnectionChecker):
+    """
+    Checks the connections by checking availability of one or many streams selected by the developer
+
+    Attributes:
+        stream_name (List[str]): names of streams to check
+    """
+
+    stream_names: List[str]
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._parameters = parameters
+
+    def check_connection(
+        self, source: AbstractSource, logger: logging.Logger, config: Mapping[str, Any]
+    ) -> Tuple[bool, Any]:
+        streams = source.streams(config=config)
+        stream_name_to_stream = {s.name: s for s in streams}
+        if len(streams) == 0:
+            return False, f"No streams to connect to from source {source}"
+        for stream_name in self.stream_names:
+            if stream_name not in stream_name_to_stream.keys():
+                raise ValueError(
+                    f"{stream_name} is not part of the catalog. Expected one of {stream_name_to_stream.keys()}."
+                )
+
+            stream = stream_name_to_stream[stream_name]
+            availability_strategy = HttpAvailabilityStrategy()
+            try:
+                stream_is_available, reason = availability_strategy.check_availability(
+                    stream, logger
+                )
+                if not stream_is_available:
+                    return False, reason
+            except Exception as error:
+                logger.error(
+                    f"Encountered an error trying to connect to stream {stream_name}. Error: \n {traceback.format_exc()}"
+                )
+                return False, f"Unable to connect to stream {stream_name} - {error}"
+        return True, None

airbyte_cdk/sources/declarative/checks/connection_checker.py

@@ -0,0 +1,35 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Mapping, Tuple
+
+from airbyte_cdk import AbstractSource
+
+
+class ConnectionChecker(ABC):
+    """
+    Abstract base class for checking a connection
+    """
+
+    @abstractmethod
+    def check_connection(
+        self, source: AbstractSource, logger: logging.Logger, config: Mapping[str, Any]
+    ) -> Tuple[bool, Any]:
+        """
+        Tests if the input configuration can be used to successfully connect to the integration e.g: if a provided Stripe API token can be used to connect
+        to the Stripe API.
+
+        :param source: source
+        :param logger: source logger
+        :param config: The user-provided configuration as specified by the source's spec.
+          This usually contains information required to check connection e.g. tokens, secrets and keys etc.
+        :return: A tuple of (boolean, error). If boolean is true, then the connection check is successful
+          and we can connect to the underlying data source using the provided configuration.
+          Otherwise, the input config cannot be used to connect to the underlying data source,
+          and the "error" object should describe what went wrong.
+          The error object will be cast to string to display the problem to the user.
+        """
+        pass

airbyte_cdk/sources/declarative/concurrency_level/__init__.py

@@ -0,0 +1,7 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.concurrency_level.concurrency_level import ConcurrencyLevel
+
+__all__ = ["ConcurrencyLevel"]

airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py

@@ -0,0 +1,50 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, Optional, Union
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.types import Config
+
+
+@dataclass
+class ConcurrencyLevel:
+    """
+    Returns the number of worker threads that should be used when syncing concurrent streams in parallel
+
+    Attributes:
+        default_concurrency (Union[int, str]): The hardcoded integer or interpolation of how many worker threads to use during a sync
+        max_concurrency (Optional[int]): The maximum number of worker threads to use when the default_concurrency is exceeded
+    """
+
+    default_concurrency: Union[int, str]
+    max_concurrency: Optional[int]
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if isinstance(self.default_concurrency, int):
+            self._default_concurrency: Union[int, InterpolatedString] = self.default_concurrency
+        elif "config" in self.default_concurrency and not self.max_concurrency:
+            raise ValueError(
+                "ConcurrencyLevel requires that max_concurrency be defined if the default_concurrency can be used-specified"
+            )
+        else:
+            self._default_concurrency = InterpolatedString.create(
+                self.default_concurrency, parameters=parameters
+            )
+
+    def get_concurrency_level(self) -> int:
+        if isinstance(self._default_concurrency, InterpolatedString):
+            evaluated_default_concurrency = self._default_concurrency.eval(config=self.config)
+            if not isinstance(evaluated_default_concurrency, int):
+                raise ValueError("default_concurrency did not evaluate to an integer")
+            return (
+                min(evaluated_default_concurrency, self.max_concurrency)
+                if self.max_concurrency
+                else evaluated_default_concurrency
+            )
+        else:
+            return self._default_concurrency