airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py

@@ -0,0 +1,119 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Mapping, MutableMapping, Optional, Union
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping import NestedMapping
+from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_nested_request_input_provider import (
+    InterpolatedNestedRequestInputProvider,
+)
+from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_input_provider import (
+    InterpolatedRequestInputProvider,
+)
+from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
+    RequestOptionsProvider,
+)
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+
+RequestInput = Union[str, Mapping[str, str]]
+ValidRequestTypes = (str, list)
+
+
+@dataclass
+class InterpolatedRequestOptionsProvider(RequestOptionsProvider):
+    """
+    Defines the request options to set on an outgoing HTTP request by evaluating `InterpolatedMapping`s
+
+    Attributes:
+        config (Config): The user-provided configuration as specified by the source's spec
+        request_parameters (Union[str, Mapping[str, str]]): The request parameters to set on an outgoing HTTP request
+        request_headers (Union[str, Mapping[str, str]]): The request headers to set on an outgoing HTTP request
+        request_body_data (Union[str, Mapping[str, str]]): The body data to set on an outgoing HTTP request
+        request_body_json (Union[str, Mapping[str, str]]): The json content to set on an outgoing HTTP request
+    """
+
+    parameters: InitVar[Mapping[str, Any]]
+    config: Config = field(default_factory=dict)
+    request_parameters: Optional[RequestInput] = None
+    request_headers: Optional[RequestInput] = None
+    request_body_data: Optional[RequestInput] = None
+    request_body_json: Optional[NestedMapping] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if self.request_parameters is None:
+            self.request_parameters = {}
+        if self.request_headers is None:
+            self.request_headers = {}
+        if self.request_body_data is None:
+            self.request_body_data = {}
+        if self.request_body_json is None:
+            self.request_body_json = {}
+
+        if self.request_body_json and self.request_body_data:
+            raise ValueError(
+                "RequestOptionsProvider should only contain either 'request_body_data' or 'request_body_json' not both"
+            )
+
+        self._parameter_interpolator = InterpolatedRequestInputProvider(
+            config=self.config, request_inputs=self.request_parameters, parameters=parameters
+        )
+        self._headers_interpolator = InterpolatedRequestInputProvider(
+            config=self.config, request_inputs=self.request_headers, parameters=parameters
+        )
+        self._body_data_interpolator = InterpolatedRequestInputProvider(
+            config=self.config, request_inputs=self.request_body_data, parameters=parameters
+        )
+        self._body_json_interpolator = InterpolatedNestedRequestInputProvider(
+            config=self.config, request_inputs=self.request_body_json, parameters=parameters
+        )
+
+    def get_request_params(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> MutableMapping[str, Any]:
+        interpolated_value = self._parameter_interpolator.eval_request_inputs(
+            stream_slice,
+            next_page_token,
+            valid_key_types=(str,),
+            valid_value_types=ValidRequestTypes,
+        )
+        if isinstance(interpolated_value, dict):
+            return interpolated_value
+        return {}
+
+    def get_request_headers(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return self._headers_interpolator.eval_request_inputs(stream_slice, next_page_token)
+
+    def get_request_body_data(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Union[Mapping[str, Any], str]:
+        return self._body_data_interpolator.eval_request_inputs(
+            stream_slice,
+            next_page_token,
+            valid_key_types=(str,),
+            valid_value_types=ValidRequestTypes,
+        )
+
+    def get_request_body_json(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return self._body_json_interpolator.eval_request_inputs(stream_slice, next_page_token)

airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py

@@ -0,0 +1,79 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Any, Mapping, Optional, Union
+
+from airbyte_cdk.sources.types import StreamSlice, StreamState
+
+
+@dataclass
+class RequestOptionsProvider:
+    """
+    Defines the request options to set on an outgoing HTTP request
+
+    Options can be passed by
+    - request parameter
+    - request headers
+    - body data
+    - json content
+    """
+
+    @abstractmethod
+    def get_request_params(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        """
+        Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.
+
+        E.g: you might want to define query parameters for paging if next_page_token is not None.
+        """
+        pass
+
+    @abstractmethod
+    def get_request_headers(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        """Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method."""
+
+    @abstractmethod
+    def get_request_body_data(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Union[Mapping[str, Any], str]:
+        """
+        Specifies how to populate the body of the request with a non-JSON payload.
+
+        If returns a ready text that it will be sent as is.
+        If returns a dict that it will be converted to a urlencoded form.
+        E.g. {"key1": "value1", "key2": "value2"} => "key1=value1&key2=value2"
+
+        At the same time only one of the 'request_body_data' and 'request_body_json' functions can be overridden.
+        """
+
+    @abstractmethod
+    def get_request_body_json(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        """
+        Specifies how to populate the body of the request with a JSON payload.
+
+        At the same time only one of the 'request_body_data' and 'request_body_json' functions can be overridden.
+        """

airbyte_cdk/sources/declarative/requesters/request_path.py

@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping
+
+
+@dataclass
+class RequestPath:
+    """
+    Describes that a component value should be inserted into the path
+    """
+
+    parameters: InitVar[Mapping[str, Any]]

airbyte_cdk/sources/declarative/requesters/requester.py

@@ -0,0 +1,144 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import abstractmethod
+from enum import Enum
+from typing import Any, Callable, Mapping, MutableMapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
+from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
+    RequestOptionsProvider,
+)
+from airbyte_cdk.sources.types import StreamSlice, StreamState
+
+
+class HttpMethod(Enum):
+    """
+    Http Method to use when submitting an outgoing HTTP request
+    """
+
+    DELETE = "DELETE"
+    GET = "GET"
+    PATCH = "PATCH"
+    POST = "POST"
+
+
+class Requester(RequestOptionsProvider):
+    @abstractmethod
+    def get_authenticator(self) -> DeclarativeAuthenticator:
+        """
+        Specifies the authenticator to use when submitting requests
+        """
+        pass
+
+    @abstractmethod
+    def get_url_base(
+        self,
+        *,
+        stream_state: Optional[StreamState],
+        stream_slice: Optional[StreamSlice],
+        next_page_token: Optional[Mapping[str, Any]],
+    ) -> str:
+        """
+        :return: URL base for the  API endpoint e.g: if you wanted to hit https://myapi.com/v1/some_entity then this should return "https://myapi.com/v1/"
+        """
+
+    @abstractmethod
+    def get_path(
+        self,
+        *,
+        stream_state: Optional[StreamState],
+        stream_slice: Optional[StreamSlice],
+        next_page_token: Optional[Mapping[str, Any]],
+    ) -> str:
+        """
+        Returns the URL path for the API endpoint e.g: if you wanted to hit https://myapi.com/v1/some_entity then this should return "some_entity"
+        """
+
+    @abstractmethod
+    def get_method(self) -> HttpMethod:
+        """
+        Specifies the HTTP method to use
+        """
+
+    @abstractmethod
+    def get_request_params(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> MutableMapping[str, Any]:
+        """
+        Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.
+
+        E.g: you might want to define query parameters for paging if next_page_token is not None.
+        """
+
+    @abstractmethod
+    def get_request_headers(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        """
+        Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.
+        """
+
+    @abstractmethod
+    def get_request_body_data(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Union[Mapping[str, Any], str]:
+        """
+        Specifies how to populate the body of the request with a non-JSON payload.
+
+        If returns a ready text that it will be sent as is.
+        If returns a dict that it will be converted to a urlencoded form.
+        E.g. {"key1": "value1", "key2": "value2"} => "key1=value1&key2=value2"
+
+        At the same time only one of the 'request_body_data' and 'request_body_json' functions can be overridden.
+        """
+
+    @abstractmethod
+    def get_request_body_json(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        """
+        Specifies how to populate the body of the request with a JSON payload.
+
+        At the same time only one of the 'request_body_data' and 'request_body_json' functions can be overridden.
+        """
+
+    @abstractmethod
+    def send_request(
+        self,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+        path: Optional[str] = None,
+        request_headers: Optional[Mapping[str, Any]] = None,
+        request_params: Optional[Mapping[str, Any]] = None,
+        request_body_data: Optional[Union[Mapping[str, Any], str]] = None,
+        request_body_json: Optional[Mapping[str, Any]] = None,
+        log_formatter: Optional[Callable[[requests.Response], Any]] = None,
+    ) -> Optional[requests.Response]:
+        """
+        Sends a request and returns the response. Might return no response if the error handler chooses to ignore the response or throw an exception in case of an error.
+        If path is set, the path configured on the requester itself is ignored.
+        If header, params and body are set, they are merged with the ones configured on the requester itself.
+
+        If a log formatter is provided, it's used to log the performed request and response. If it's not provided, no logging is performed.
+        """

airbyte_cdk/sources/declarative/resolvers/__init__.py

@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Mapping
+
+from pydantic.v1 import BaseModel
+
+from airbyte_cdk.sources.declarative.models import (
+    ConfigComponentsResolver as ConfigComponentsResolverModel,
+)
+from airbyte_cdk.sources.declarative.models import (
+    HttpComponentsResolver as HttpComponentsResolverModel,
+)
+from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
+    ComponentMappingDefinition,
+    ComponentsResolver,
+    ResolvedComponentMappingDefinition,
+)
+from airbyte_cdk.sources.declarative.resolvers.config_components_resolver import (
+    ConfigComponentsResolver,
+    StreamConfig,
+)
+from airbyte_cdk.sources.declarative.resolvers.http_components_resolver import (
+    HttpComponentsResolver,
+)
+
+COMPONENTS_RESOLVER_TYPE_MAPPING: Mapping[str, type[BaseModel]] = {
+    "HttpComponentsResolver": HttpComponentsResolverModel,
+    "ConfigComponentsResolver": ConfigComponentsResolverModel,
+}
+
+__all__ = [
+    "ComponentsResolver",
+    "HttpComponentsResolver",
+    "ComponentMappingDefinition",
+    "ResolvedComponentMappingDefinition",
+    "StreamConfig",
+    "ConfigComponentsResolver",
+    "COMPONENTS_RESOLVER_TYPE_MAPPING",
+]

airbyte_cdk/sources/declarative/resolvers/components_resolver.py

@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from dataclasses import InitVar, dataclass
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Type, Union
+
+from typing_extensions import deprecated
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.source import ExperimentalClassWarning
+
+
+@dataclass(frozen=True)
+class ComponentMappingDefinition:
+    """Defines the configuration for mapping a component in a stream. This class specifies
+    what field in the stream template should be updated with value, supporting dynamic interpolation
+    and type enforcement."""
+
+    field_path: List["InterpolatedString"]
+    value: Union["InterpolatedString", str]
+    value_type: Optional[Type[Any]]
+    parameters: InitVar[Mapping[str, Any]]
+
+
+@dataclass(frozen=True)
+class ResolvedComponentMappingDefinition:
+    """Defines resolved configuration for mapping a component in a stream. This class specifies
+    what field in the stream template should be updated with value, supporting dynamic interpolation
+    and type enforcement."""
+
+    field_path: List["InterpolatedString"]
+    value: "InterpolatedString"
+    value_type: Optional[Type[Any]]
+    parameters: InitVar[Mapping[str, Any]]
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class ComponentsResolver(ABC):
+    """
+    Abstract base class for resolving components in a stream template.
+    """
+
+    @abstractmethod
+    def resolve_components(
+        self, stream_template_config: Dict[str, Any]
+    ) -> Iterable[Dict[str, Any]]:
+        """
+        Maps and populates values into a stream template configuration.
+        :param stream_template_config: The stream template with placeholders for components.
+        :yields: The resolved stream config with populated values.
+        """
+        pass

airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py

@@ -0,0 +1,136 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from copy import deepcopy
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Dict, Iterable, List, Mapping, Union
+
+import dpath
+from typing_extensions import deprecated
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
+    ComponentMappingDefinition,
+    ComponentsResolver,
+    ResolvedComponentMappingDefinition,
+)
+from airbyte_cdk.sources.source import ExperimentalClassWarning
+from airbyte_cdk.sources.types import Config
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class StreamConfig:
+    """
+    Identifies stream config details for dynamic schema extraction and processing.
+    """
+
+    configs_pointer: List[Union[InterpolatedString, str]]
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self.configs_pointer = [
+            InterpolatedString.create(path, parameters=parameters) for path in self.configs_pointer
+        ]
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class ConfigComponentsResolver(ComponentsResolver):
+    """
+    Resolves and populates stream templates with components fetched via source config.
+
+    Attributes:
+        stream_config (StreamConfig): The description of stream configuration used to fetch stream config from source config.
+        config (Config): Configuration object for the resolver.
+        components_mapping (List[ComponentMappingDefinition]): List of mappings to resolve.
+        parameters (InitVar[Mapping[str, Any]]): Additional parameters for interpolation.
+    """
+
+    stream_config: StreamConfig
+    config: Config
+    components_mapping: List[ComponentMappingDefinition]
+    parameters: InitVar[Mapping[str, Any]]
+    _resolved_components: List[ResolvedComponentMappingDefinition] = field(
+        init=False, repr=False, default_factory=list
+    )
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        """
+        Initializes and parses component mappings, converting them to resolved definitions.
+
+        Args:
+            parameters (Mapping[str, Any]): Parameters for interpolation.
+        """
+
+        for component_mapping in self.components_mapping:
+            if isinstance(component_mapping.value, (str, InterpolatedString)):
+                interpolated_value = (
+                    InterpolatedString.create(component_mapping.value, parameters=parameters)
+                    if isinstance(component_mapping.value, str)
+                    else component_mapping.value
+                )
+
+                field_path = [
+                    InterpolatedString.create(path, parameters=parameters)
+                    for path in component_mapping.field_path
+                ]
+
+                self._resolved_components.append(
+                    ResolvedComponentMappingDefinition(
+                        field_path=field_path,
+                        value=interpolated_value,
+                        value_type=component_mapping.value_type,
+                        parameters=parameters,
+                    )
+                )
+            else:
+                raise ValueError(
+                    f"Expected a string or InterpolatedString for value in mapping: {component_mapping}"
+                )
+
+    @property
+    def _stream_config(self) -> Iterable[Mapping[str, Any]]:
+        path = [
+            node.eval(self.config) if not isinstance(node, str) else node
+            for node in self.stream_config.configs_pointer
+        ]
+        stream_config = dpath.get(dict(self.config), path, default=[])
+
+        if not isinstance(stream_config, list):
+            stream_config = [stream_config]
+
+        return stream_config
+
+    def resolve_components(
+        self, stream_template_config: Dict[str, Any]
+    ) -> Iterable[Dict[str, Any]]:
+        """
+        Resolves components in the stream template configuration by populating values.
+
+        Args:
+            stream_template_config (Dict[str, Any]): Stream template to populate.
+
+        Yields:
+            Dict[str, Any]: Updated configurations with resolved components.
+        """
+        kwargs = {"stream_template_config": stream_template_config}
+
+        for components_values in self._stream_config:
+            updated_config = deepcopy(stream_template_config)
+            kwargs["components_values"] = components_values  # type: ignore[assignment] # component_values will always be of type Mapping[str, Any]
+
+            for resolved_component in self._resolved_components:
+                valid_types = (
+                    (resolved_component.value_type,) if resolved_component.value_type else None
+                )
+                value = resolved_component.value.eval(
+                    self.config, valid_types=valid_types, **kwargs
+                )
+
+                path = [path.eval(self.config, **kwargs) for path in resolved_component.field_path]
+
+                dpath.set(updated_config, path, value)
+
+            yield updated_config