airbyte-cdk 6.8.3rc1__py3-none-any.whl → 6.9.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -119,6 +119,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     CheckStream as CheckStreamModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    ComponentMappingDefinition as ComponentMappingDefinitionModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     CompositeErrorHandler as CompositeErrorHandlerModel,
 )
@@ -185,12 +188,18 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     DpathExtractor as DpathExtractorModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    DynamicSchemaLoader as DynamicSchemaLoaderModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     GzipJsonDecoder as GzipJsonDecoderModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    HttpComponentsResolver as HttpComponentsResolverModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     HttpRequester as HttpRequesterModel,
 )
@@ -272,6 +281,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ResponseToFileExtractor as ResponseToFileExtractorModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    SchemaTypeIdentifier as SchemaTypeIdentifierModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     SelectiveAuthenticator as SelectiveAuthenticatorModel,
 )
@@ -285,6 +297,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     SubstreamPartitionRouter as SubstreamPartitionRouterModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    TypesMap as TypesMapModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     WaitTimeFromHeader as WaitTimeFromHeaderModel,
@@ -298,6 +313,7 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.partition_routers import (
     CartesianProductStreamSlicer,
     ListPartitionRouter,
+    PartitionRouter,
     SinglePartitionRouter,
     SubstreamPartitionRouter,
 )
@@ -338,6 +354,10 @@ from airbyte_cdk.sources.declarative.requesters.request_options import (
 )
 from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
 from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
+from airbyte_cdk.sources.declarative.resolvers import (
+    ComponentMappingDefinition,
+    HttpComponentsResolver,
+)
 from airbyte_cdk.sources.declarative.retrievers import (
     AsyncRetriever,
     SimpleRetriever,
@@ -345,8 +365,11 @@ from airbyte_cdk.sources.declarative.retrievers import (
 )
 from airbyte_cdk.sources.declarative.schema import (
     DefaultSchemaLoader,
+    DynamicSchemaLoader,
     InlineSchemaLoader,
     JsonFileSchemaLoader,
+    SchemaTypeIdentifier,
+    TypesMap,
 )
 from airbyte_cdk.sources.declarative.spec import Spec
 from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer
@@ -444,6 +467,9 @@ class ModelToComponentFactory:
             IterableDecoderModel: self.create_iterable_decoder,
             XmlDecoderModel: self.create_xml_decoder,
             JsonFileSchemaLoaderModel: self.create_json_file_schema_loader,
+            DynamicSchemaLoaderModel: self.create_dynamic_schema_loader,
+            SchemaTypeIdentifierModel: self.create_schema_type_identifier,
+            TypesMapModel: self.create_types_map,
             JwtAuthenticatorModel: self.create_jwt_authenticator,
             LegacyToPerPartitionStateMigrationModel: self.create_legacy_to_per_partition_state_migration,
             ListPartitionRouterModel: self.create_list_partition_router,
@@ -467,6 +493,8 @@ class ModelToComponentFactory:
             WaitTimeFromHeaderModel: self.create_wait_time_from_header,
             WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header,
             AsyncRetrieverModel: self.create_async_retriever,
+            HttpComponentsResolverModel: self.create_http_components_resolver,
+            ComponentMappingDefinitionModel: self.create_components_mapping_definition,
         }
 
         # Needed for the case where we need to perform a second parse on the fields of a custom component
@@ -1281,19 +1309,20 @@ class ModelToComponentFactory:
             parameters=model.parameters or {},
         )
 
-    def _merge_stream_slicers(
-        self, model: DeclarativeStreamModel, config: Config
-    ) -> Optional[StreamSlicer]:
-        stream_slicer = None
+    def _build_stream_slicer_from_partition_router(
+        self,
+        model: Union[AsyncRetrieverModel, CustomRetrieverModel, SimpleRetrieverModel],
+        config: Config,
+    ) -> Optional[PartitionRouter]:
         if (
-            hasattr(model.retriever, "partition_router")
-            and isinstance(model.retriever, SimpleRetrieverModel)
-            and model.retriever.partition_router
+            hasattr(model, "partition_router")
+            and isinstance(model, SimpleRetrieverModel)
+            and model.partition_router
         ):
-            stream_slicer_model = model.retriever.partition_router
+            stream_slicer_model = model.partition_router
 
             if isinstance(stream_slicer_model, list):
-                stream_slicer = CartesianProductStreamSlicer(
+                return CartesianProductStreamSlicer(
                     [
                         self._create_component_from_model(model=slicer, config=config)
                         for slicer in stream_slicer_model
@@ -1301,9 +1330,24 @@ class ModelToComponentFactory:
                     parameters={},
                 )
             else:
-                stream_slicer = self._create_component_from_model(
-                    model=stream_slicer_model, config=config
-                )
+                return self._create_component_from_model(model=stream_slicer_model, config=config)  # type: ignore[no-any-return]
+                # Will be created PartitionRouter as stream_slicer_model is model.partition_router
+        return None
+
+    def _build_resumable_cursor_from_paginator(
+        self,
+        model: Union[AsyncRetrieverModel, CustomRetrieverModel, SimpleRetrieverModel],
+        stream_slicer: Optional[StreamSlicer],
+    ) -> Optional[StreamSlicer]:
+        if hasattr(model, "paginator") and model.paginator and not stream_slicer:
+            # For the regular Full-Refresh streams, we use the high level `ResumableFullRefreshCursor`
+            return ResumableFullRefreshCursor(parameters={})
+        return None
+
+    def _merge_stream_slicers(
+        self, model: DeclarativeStreamModel, config: Config
+    ) -> Optional[StreamSlicer]:
+        stream_slicer = self._build_stream_slicer_from_partition_router(model.retriever, config)
 
         if model.incremental_sync and stream_slicer:
             incremental_sync_model = model.incremental_sync
@@ -1346,15 +1390,7 @@ class ModelToComponentFactory:
                 ),
                 partition_router=stream_slicer,
             )
-        elif (
-            hasattr(model.retriever, "paginator")
-            and model.retriever.paginator
-            and not stream_slicer
-        ):
-            # For the regular Full-Refresh streams, we use the high level `ResumableFullRefreshCursor`
-            return ResumableFullRefreshCursor(parameters={})
-        else:
-            return None
+        return self._build_resumable_cursor_from_paginator(model.retriever, stream_slicer)
 
     def create_default_error_handler(
         self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any
@@ -1553,6 +1589,63 @@ class ModelToComponentFactory:
     ) -> InlineSchemaLoader:
         return InlineSchemaLoader(schema=model.schema_ or {}, parameters={})
 
+    @staticmethod
+    def create_types_map(model: TypesMapModel, **kwargs: Any) -> TypesMap:
+        return TypesMap(target_type=model.target_type, current_type=model.current_type)
+
+    def create_schema_type_identifier(
+        self, model: SchemaTypeIdentifierModel, config: Config, **kwargs: Any
+    ) -> SchemaTypeIdentifier:
+        types_mapping = []
+        if model.types_mapping:
+            types_mapping.extend(
+                [
+                    self._create_component_from_model(types_map, config=config)
+                    for types_map in model.types_mapping
+                ]
+            )
+        model_schema_pointer: List[Union[InterpolatedString, str]] = (
+            [x for x in model.schema_pointer] if model.schema_pointer else []
+        )
+        model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer]
+        model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = (
+            [x for x in model.type_pointer] if model.type_pointer else None
+        )
+
+        return SchemaTypeIdentifier(
+            schema_pointer=model_schema_pointer,
+            key_pointer=model_key_pointer,
+            type_pointer=model_type_pointer,
+            types_mapping=types_mapping,
+            parameters=model.parameters or {},
+        )
+
+    def create_dynamic_schema_loader(
+        self, model: DynamicSchemaLoaderModel, config: Config, **kwargs: Any
+    ) -> DynamicSchemaLoader:
+        stream_slicer = self._build_stream_slicer_from_partition_router(model.retriever, config)
+        combined_slicers = self._build_resumable_cursor_from_paginator(
+            model.retriever, stream_slicer
+        )
+
+        retriever = self._create_component_from_model(
+            model=model.retriever,
+            config=config,
+            name="",
+            primary_key=None,
+            stream_slicer=combined_slicers,
+            transformations=[],
+        )
+        schema_type_identifier = self._create_component_from_model(
+            model.schema_type_identifier, config=config, parameters=model.parameters or {}
+        )
+        return DynamicSchemaLoader(
+            retriever=retriever,
+            config=config,
+            schema_type_identifier=schema_type_identifier,
+            parameters=model.parameters or {},
+        )
+
     @staticmethod
     def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> JsonDecoder:
         return JsonDecoder(parameters={})
@@ -2218,3 +2311,56 @@ class ModelToComponentFactory:
 
     def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level:
         return Level.DEBUG if emit_connector_builder_messages else Level.INFO
+
+    @staticmethod
+    def create_components_mapping_definition(
+        model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any
+    ) -> ComponentMappingDefinition:
+        interpolated_value = InterpolatedString.create(
+            model.value, parameters=model.parameters or {}
+        )
+        field_path = [
+            InterpolatedString.create(path, parameters=model.parameters or {})
+            for path in model.field_path
+        ]
+        return ComponentMappingDefinition(
+            field_path=field_path,  # type: ignore[arg-type] # field_path can be str and InterpolatedString
+            value=interpolated_value,
+            value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type),
+            parameters=model.parameters or {},
+        )
+
+    def create_http_components_resolver(
+        self, model: HttpComponentsResolverModel, config: Config
+    ) -> Any:
+        stream_slicer = self._build_stream_slicer_from_partition_router(model.retriever, config)
+        combined_slicers = self._build_resumable_cursor_from_paginator(
+            model.retriever, stream_slicer
+        )
+
+        retriever = self._create_component_from_model(
+            model=model.retriever,
+            config=config,
+            name="",
+            primary_key=None,
+            stream_slicer=combined_slicers,
+            transformations=[],
+        )
+
+        components_mapping = [
+            self._create_component_from_model(
+                model=components_mapping_definition_model,
+                value_type=ModelToComponentFactory._json_schema_type_name_to_type(
+                    components_mapping_definition_model.value_type
+                ),
+                config=config,
+            )
+            for components_mapping_definition_model in model.components_mapping
+        ]
+
+        return HttpComponentsResolver(
+            retriever=retriever,
+            config=config,
+            components_mapping=components_mapping,
+            parameters=model.parameters or {},
+        )

airbyte_cdk/sources/declarative/partition_routers/__init__.py

@@ -6,5 +6,6 @@ from airbyte_cdk.sources.declarative.partition_routers.cartesian_product_stream_
 from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import ListPartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import SinglePartitionRouter
 from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter
+from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
 
-__all__ = ["CartesianProductStreamSlicer", "ListPartitionRouter", "SinglePartitionRouter", "SubstreamPartitionRouter"]
+__all__ = ["CartesianProductStreamSlicer", "ListPartitionRouter", "SinglePartitionRouter", "SubstreamPartitionRouter", "PartitionRouter"]

airbyte_cdk/sources/declarative/resolvers/__init__.py

@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.resolvers.components_resolver import ComponentsResolver, ComponentMappingDefinition, ResolvedComponentMappingDefinition
+from airbyte_cdk.sources.declarative.resolvers.http_components_resolver import HttpComponentsResolver
+from airbyte_cdk.sources.declarative.models import HttpComponentsResolver as HttpComponentsResolverModel
+
+COMPONENTS_RESOLVER_TYPE_MAPPING = {
+    "HttpComponentsResolver": HttpComponentsResolverModel
+}
+
+__all__ = ["ComponentsResolver", "HttpComponentsResolver", "ComponentMappingDefinition", "ResolvedComponentMappingDefinition", "COMPONENTS_RESOLVER_TYPE_MAPPING"]

airbyte_cdk/sources/declarative/resolvers/components_resolver.py

@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from dataclasses import InitVar, dataclass
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Type, Union
+
+from typing_extensions import deprecated
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.source import ExperimentalClassWarning
+
+
+@dataclass(frozen=True)
+class ComponentMappingDefinition:
+    """Defines the configuration for mapping a component in a stream. This class specifies
+    what field in the stream template should be updated with value, supporting dynamic interpolation
+    and type enforcement."""
+
+    field_path: List["InterpolatedString"]
+    value: Union["InterpolatedString", str]
+    value_type: Optional[Type[Any]]
+    parameters: InitVar[Mapping[str, Any]]
+
+
+@dataclass(frozen=True)
+class ResolvedComponentMappingDefinition:
+    """Defines resolved configuration for mapping a component in a stream. This class specifies
+    what field in the stream template should be updated with value, supporting dynamic interpolation
+    and type enforcement."""
+
+    field_path: List["InterpolatedString"]
+    value: "InterpolatedString"
+    value_type: Optional[Type[Any]]
+    parameters: InitVar[Mapping[str, Any]]
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class ComponentsResolver(ABC):
+    """
+    Abstract base class for resolving components in a stream template.
+    """
+
+    @abstractmethod
+    def resolve_components(
+        self, stream_template_config: Dict[str, Any]
+    ) -> Iterable[Dict[str, Any]]:
+        """
+        Maps and populates values into a stream template configuration.
+        :param stream_template_config: The stream template with placeholders for components.
+        :yields: The resolved stream config with populated values.
+        """
+        pass

airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py

@@ -0,0 +1,106 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from copy import deepcopy
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Dict, Iterable, List, Mapping
+
+import dpath
+from typing_extensions import deprecated
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
+    ComponentMappingDefinition,
+    ComponentsResolver,
+    ResolvedComponentMappingDefinition,
+)
+from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
+from airbyte_cdk.sources.source import ExperimentalClassWarning
+from airbyte_cdk.sources.types import Config
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class HttpComponentsResolver(ComponentsResolver):
+    """
+    Resolves and populates stream templates with components fetched via an HTTP retriever.
+
+    Attributes:
+        retriever (Retriever): The retriever used to fetch data from an API.
+        config (Config): Configuration object for the resolver.
+        components_mapping (List[ComponentMappingDefinition]): List of mappings to resolve.
+        parameters (InitVar[Mapping[str, Any]]): Additional parameters for interpolation.
+    """
+
+    retriever: Retriever
+    config: Config
+    components_mapping: List[ComponentMappingDefinition]
+    parameters: InitVar[Mapping[str, Any]]
+    _resolved_components: List[ResolvedComponentMappingDefinition] = field(
+        init=False, repr=False, default_factory=list
+    )
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        """
+        Initializes and parses component mappings, converting them to resolved definitions.
+
+        Args:
+            parameters (Mapping[str, Any]): Parameters for interpolation.
+        """
+        for component_mapping in self.components_mapping:
+            if isinstance(component_mapping.value, (str, InterpolatedString)):
+                interpolated_value = (
+                    InterpolatedString.create(component_mapping.value, parameters=parameters)
+                    if isinstance(component_mapping.value, str)
+                    else component_mapping.value
+                )
+
+                field_path = [
+                    InterpolatedString.create(path, parameters=parameters)
+                    for path in component_mapping.field_path
+                ]
+
+                self._resolved_components.append(
+                    ResolvedComponentMappingDefinition(
+                        field_path=field_path,
+                        value=interpolated_value,
+                        value_type=component_mapping.value_type,
+                        parameters=parameters,
+                    )
+                )
+            else:
+                raise ValueError(
+                    f"Expected a string or InterpolatedString for value in mapping: {component_mapping}"
+                )
+
+    def resolve_components(
+        self, stream_template_config: Dict[str, Any]
+    ) -> Iterable[Dict[str, Any]]:
+        """
+        Resolves components in the stream template configuration by populating values.
+
+        Args:
+            stream_template_config (Dict[str, Any]): Stream template to populate.
+
+        Yields:
+            Dict[str, Any]: Updated configurations with resolved components.
+        """
+        kwargs = {"stream_template_config": stream_template_config}
+
+        for components_values in self.retriever.read_records({}):
+            updated_config = deepcopy(stream_template_config)
+            kwargs["components_values"] = components_values  # type: ignore[assignment] # component_values will always be of type Mapping[str, Any]
+
+            for resolved_component in self._resolved_components:
+                valid_types = (
+                    (resolved_component.value_type,) if resolved_component.value_type else None
+                )
+                value = resolved_component.value.eval(
+                    self.config, valid_types=valid_types, **kwargs
+                )
+
+                path = [path.eval(self.config, **kwargs) for path in resolved_component.field_path]
+                dpath.set(updated_config, path, value)
+
+            yield updated_config

airbyte_cdk/sources/declarative/schema/__init__.py

@@ -6,5 +6,6 @@ from airbyte_cdk.sources.declarative.schema.default_schema_loader import Default
 from airbyte_cdk.sources.declarative.schema.inline_schema_loader import InlineSchemaLoader
 from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader
 from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
+from airbyte_cdk.sources.declarative.schema.dynamic_schema_loader import DynamicSchemaLoader, TypesMap, SchemaTypeIdentifier
 
-__all__ = ["JsonFileSchemaLoader", "DefaultSchemaLoader", "SchemaLoader", "InlineSchemaLoader"]
+__all__ = ["JsonFileSchemaLoader", "DefaultSchemaLoader", "SchemaLoader", "InlineSchemaLoader", "DynamicSchemaLoader", "TypesMap", "SchemaTypeIdentifier"]