airbyte-cdk 0.72.1__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

airbyte_cdk/sources/declarative/resolvers/components_resolver.py

@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from dataclasses import InitVar, dataclass
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Type, Union
+
+from typing_extensions import deprecated
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.source import ExperimentalClassWarning
+
+
+@dataclass(frozen=True)
+class ComponentMappingDefinition:
+    """Defines the configuration for mapping a component in a stream. This class specifies
+    what field in the stream template should be updated with value, supporting dynamic interpolation
+    and type enforcement."""
+
+    field_path: List["InterpolatedString"]
+    value: Union["InterpolatedString", str]
+    value_type: Optional[Type[Any]]
+    parameters: InitVar[Mapping[str, Any]]
+
+
+@dataclass(frozen=True)
+class ResolvedComponentMappingDefinition:
+    """Defines resolved configuration for mapping a component in a stream. This class specifies
+    what field in the stream template should be updated with value, supporting dynamic interpolation
+    and type enforcement."""
+
+    field_path: List["InterpolatedString"]
+    value: "InterpolatedString"
+    value_type: Optional[Type[Any]]
+    parameters: InitVar[Mapping[str, Any]]
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class ComponentsResolver(ABC):
+    """
+    Abstract base class for resolving components in a stream template.
+    """
+
+    @abstractmethod
+    def resolve_components(
+        self, stream_template_config: Dict[str, Any]
+    ) -> Iterable[Dict[str, Any]]:
+        """
+        Maps and populates values into a stream template configuration.
+        :param stream_template_config: The stream template with placeholders for components.
+        :yields: The resolved stream config with populated values.
+        """
+        pass

airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py

@@ -0,0 +1,136 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from copy import deepcopy
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Dict, Iterable, List, Mapping, Union
+
+import dpath
+from typing_extensions import deprecated
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
+    ComponentMappingDefinition,
+    ComponentsResolver,
+    ResolvedComponentMappingDefinition,
+)
+from airbyte_cdk.sources.source import ExperimentalClassWarning
+from airbyte_cdk.sources.types import Config
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class StreamConfig:
+    """
+    Identifies stream config details for dynamic schema extraction and processing.
+    """
+
+    configs_pointer: List[Union[InterpolatedString, str]]
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self.configs_pointer = [
+            InterpolatedString.create(path, parameters=parameters) for path in self.configs_pointer
+        ]
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class ConfigComponentsResolver(ComponentsResolver):
+    """
+    Resolves and populates stream templates with components fetched via source config.
+
+    Attributes:
+        stream_config (StreamConfig): The description of stream configuration used to fetch stream config from source config.
+        config (Config): Configuration object for the resolver.
+        components_mapping (List[ComponentMappingDefinition]): List of mappings to resolve.
+        parameters (InitVar[Mapping[str, Any]]): Additional parameters for interpolation.
+    """
+
+    stream_config: StreamConfig
+    config: Config
+    components_mapping: List[ComponentMappingDefinition]
+    parameters: InitVar[Mapping[str, Any]]
+    _resolved_components: List[ResolvedComponentMappingDefinition] = field(
+        init=False, repr=False, default_factory=list
+    )
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        """
+        Initializes and parses component mappings, converting them to resolved definitions.
+
+        Args:
+            parameters (Mapping[str, Any]): Parameters for interpolation.
+        """
+
+        for component_mapping in self.components_mapping:
+            if isinstance(component_mapping.value, (str, InterpolatedString)):
+                interpolated_value = (
+                    InterpolatedString.create(component_mapping.value, parameters=parameters)
+                    if isinstance(component_mapping.value, str)
+                    else component_mapping.value
+                )
+
+                field_path = [
+                    InterpolatedString.create(path, parameters=parameters)
+                    for path in component_mapping.field_path
+                ]
+
+                self._resolved_components.append(
+                    ResolvedComponentMappingDefinition(
+                        field_path=field_path,
+                        value=interpolated_value,
+                        value_type=component_mapping.value_type,
+                        parameters=parameters,
+                    )
+                )
+            else:
+                raise ValueError(
+                    f"Expected a string or InterpolatedString for value in mapping: {component_mapping}"
+                )
+
+    @property
+    def _stream_config(self) -> Iterable[Mapping[str, Any]]:
+        path = [
+            node.eval(self.config) if not isinstance(node, str) else node
+            for node in self.stream_config.configs_pointer
+        ]
+        stream_config = dpath.get(dict(self.config), path, default=[])
+
+        if not isinstance(stream_config, list):
+            stream_config = [stream_config]
+
+        return stream_config
+
+    def resolve_components(
+        self, stream_template_config: Dict[str, Any]
+    ) -> Iterable[Dict[str, Any]]:
+        """
+        Resolves components in the stream template configuration by populating values.
+
+        Args:
+            stream_template_config (Dict[str, Any]): Stream template to populate.
+
+        Yields:
+            Dict[str, Any]: Updated configurations with resolved components.
+        """
+        kwargs = {"stream_template_config": stream_template_config}
+
+        for components_values in self._stream_config:
+            updated_config = deepcopy(stream_template_config)
+            kwargs["components_values"] = components_values  # type: ignore[assignment] # component_values will always be of type Mapping[str, Any]
+
+            for resolved_component in self._resolved_components:
+                valid_types = (
+                    (resolved_component.value_type,) if resolved_component.value_type else None
+                )
+                value = resolved_component.value.eval(
+                    self.config, valid_types=valid_types, **kwargs
+                )
+
+                path = [path.eval(self.config, **kwargs) for path in resolved_component.field_path]
+
+                dpath.set(updated_config, path, value)
+
+            yield updated_config

airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py

@@ -0,0 +1,112 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from copy import deepcopy
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Dict, Iterable, List, Mapping
+
+import dpath
+from typing_extensions import deprecated
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
+    ComponentMappingDefinition,
+    ComponentsResolver,
+    ResolvedComponentMappingDefinition,
+)
+from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
+from airbyte_cdk.sources.source import ExperimentalClassWarning
+from airbyte_cdk.sources.types import Config
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class HttpComponentsResolver(ComponentsResolver):
+    """
+    Resolves and populates stream templates with components fetched via an HTTP retriever.
+
+    Attributes:
+        retriever (Retriever): The retriever used to fetch data from an API.
+        config (Config): Configuration object for the resolver.
+        components_mapping (List[ComponentMappingDefinition]): List of mappings to resolve.
+        parameters (InitVar[Mapping[str, Any]]): Additional parameters for interpolation.
+    """
+
+    retriever: Retriever
+    config: Config
+    components_mapping: List[ComponentMappingDefinition]
+    parameters: InitVar[Mapping[str, Any]]
+    _resolved_components: List[ResolvedComponentMappingDefinition] = field(
+        init=False, repr=False, default_factory=list
+    )
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        """
+        Initializes and parses component mappings, converting them to resolved definitions.
+
+        Args:
+            parameters (Mapping[str, Any]): Parameters for interpolation.
+        """
+        for component_mapping in self.components_mapping:
+            if isinstance(component_mapping.value, (str, InterpolatedString)):
+                interpolated_value = (
+                    InterpolatedString.create(component_mapping.value, parameters=parameters)
+                    if isinstance(component_mapping.value, str)
+                    else component_mapping.value
+                )
+
+                field_path = [
+                    InterpolatedString.create(path, parameters=parameters)
+                    for path in component_mapping.field_path
+                ]
+
+                self._resolved_components.append(
+                    ResolvedComponentMappingDefinition(
+                        field_path=field_path,
+                        value=interpolated_value,
+                        value_type=component_mapping.value_type,
+                        parameters=parameters,
+                    )
+                )
+            else:
+                raise ValueError(
+                    f"Expected a string or InterpolatedString for value in mapping: {component_mapping}"
+                )
+
+    def resolve_components(
+        self, stream_template_config: Dict[str, Any]
+    ) -> Iterable[Dict[str, Any]]:
+        """
+        Resolves components in the stream template configuration by populating values.
+
+        Args:
+            stream_template_config (Dict[str, Any]): Stream template to populate.
+
+        Yields:
+            Dict[str, Any]: Updated configurations with resolved components.
+        """
+        kwargs = {"stream_template_config": stream_template_config}
+
+        for stream_slice in self.retriever.stream_slices():
+            for components_values in self.retriever.read_records(
+                records_schema={}, stream_slice=stream_slice
+            ):
+                updated_config = deepcopy(stream_template_config)
+                kwargs["components_values"] = components_values  # type: ignore[assignment] # component_values will always be of type Mapping[str, Any]
+                kwargs["stream_slice"] = stream_slice  # type: ignore[assignment] # stream_slice will always be of type Mapping[str, Any]
+
+                for resolved_component in self._resolved_components:
+                    valid_types = (
+                        (resolved_component.value_type,) if resolved_component.value_type else None
+                    )
+                    value = resolved_component.value.eval(
+                        self.config, valid_types=valid_types, **kwargs
+                    )
+
+                    path = [
+                        path.eval(self.config, **kwargs) for path in resolved_component.field_path
+                    ]
+                    dpath.set(updated_config, path, value)
+
+                yield updated_config

airbyte_cdk/sources/declarative/retrievers/__init__.py

@@ -2,7 +2,11 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+from airbyte_cdk.sources.declarative.retrievers.async_retriever import AsyncRetriever
 from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
-from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever, SimpleRetrieverTestReadDecorator
+from airbyte_cdk.sources.declarative.retrievers.simple_retriever import (
+    SimpleRetriever,
+    SimpleRetrieverTestReadDecorator,
+)
 
-__all__ = ["Retriever", "SimpleRetriever", "SimpleRetrieverTestReadDecorator"]
+__all__ = ["Retriever", "SimpleRetriever", "SimpleRetrieverTestReadDecorator", "AsyncRetriever"]

airbyte_cdk/sources/declarative/retrievers/async_retriever.py

@@ -0,0 +1,100 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Iterable, Mapping, Optional
+
+from typing_extensions import deprecated
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncPartition
+from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
+from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
+    AsyncJobPartitionRouter,
+)
+from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
+from airbyte_cdk.sources.source import ExperimentalClassWarning
+from airbyte_cdk.sources.streams.core import StreamData
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
+
+
+@deprecated(
+    "This class is experimental. Use at your own risk.",
+    category=ExperimentalClassWarning,
+)
+@dataclass
+class AsyncRetriever(Retriever):
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+    record_selector: RecordSelector
+    stream_slicer: AsyncJobPartitionRouter
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._parameters = parameters
+
+    @property
+    def state(self) -> StreamState:
+        """
+        As a first iteration for sendgrid, there is no state to be managed
+        """
+        return {}
+
+    @state.setter
+    def state(self, value: StreamState) -> None:
+        """
+        As a first iteration for sendgrid, there is no state to be managed
+        """
+        pass
+
+    def _get_stream_state(self) -> StreamState:
+        """
+        Gets the current state of the stream.
+
+        Returns:
+            StreamState: Mapping[str, Any]
+        """
+
+        return self.state
+
+    def _validate_and_get_stream_slice_partition(
+        self, stream_slice: Optional[StreamSlice] = None
+    ) -> AsyncPartition:
+        """
+        Validates the stream_slice argument and returns the partition from it.
+
+        Args:
+            stream_slice (Optional[StreamSlice]): The stream slice to validate and extract the partition from.
+
+        Returns:
+            AsyncPartition: The partition extracted from the stream_slice.
+
+        Raises:
+            AirbyteTracedException: If the stream_slice is not an instance of StreamSlice or if the partition is not present in the stream_slice.
+
+        """
+        if not isinstance(stream_slice, StreamSlice) or "partition" not in stream_slice.partition:
+            raise AirbyteTracedException(
+                message="Invalid arguments to AsyncJobRetriever.read_records: stream_slice is no optional. Please contact Airbyte Support",
+                failure_type=FailureType.system_error,
+            )
+        return stream_slice["partition"]  # type: ignore  # stream_slice["partition"] has been added as an AsyncPartition as part of stream_slices
+
+    def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
+        return self.stream_slicer.stream_slices()
+
+    def read_records(
+        self,
+        records_schema: Mapping[str, Any],
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Iterable[StreamData]:
+        stream_state: StreamState = self._get_stream_state()
+        partition: AsyncPartition = self._validate_and_get_stream_slice_partition(stream_slice)
+        records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(partition)
+
+        yield from self.record_selector.filter_and_transform(
+            all_data=records,
+            stream_state=stream_state,
+            records_schema=records_schema,
+            stream_slice=stream_slice,
+        )

airbyte_cdk/sources/declarative/retrievers/retriever.py

@@ -3,15 +3,13 @@
 #
 
 from abc import abstractmethod
-from dataclasses import dataclass
 from typing import Any, Iterable, Mapping, Optional
 
 from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import StreamSlice
-from airbyte_cdk.sources.declarative.types import StreamState
 from airbyte_cdk.sources.streams.core import StreamData
+from airbyte_cdk.sources.types import StreamState
 
 
-@dataclass
 class Retriever:
     """
     Responsible for fetching a stream's records from an HTTP API source.