airbyte-cdk 6.61.1__py3-none-any.whl → 6.61.3__py3-none-any.whl

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -1,33 +1,55 @@
-#
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 
+import json
 import logging
+import pkgutil
+from copy import deepcopy
 from dataclasses import dataclass, field
 from queue import Queue
+from types import ModuleType
 from typing import (
     Any,
     ClassVar,
+    Dict,
     Generic,
     Iterator,
     List,
     Mapping,
     MutableMapping,
     Optional,
+    Set,
     Tuple,
     Union,
 )
 
+import orjson
+import yaml
 from airbyte_protocol_dataclasses.models import Level
+from jsonschema.exceptions import ValidationError
+from jsonschema.validators import validate
 
+from airbyte_cdk.config_observation import create_connector_config_control_message
+from airbyte_cdk.connector_builder.models import (
+    LogMessage as ConnectorBuilderLogMessage,
+)
+from airbyte_cdk.manifest_migrations.migration_handler import (
+    ManifestMigrationHandler,
+)
 from airbyte_cdk.models import (
     AirbyteCatalog,
+    AirbyteConnectionStatus,
     AirbyteMessage,
     AirbyteStateMessage,
     ConfiguredAirbyteCatalog,
+    ConnectorSpecification,
+    FailureType,
 )
+from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerializer
+from airbyte_cdk.sources.abstract_source import AbstractSource
 from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
+from airbyte_cdk.sources.declarative.checks import COMPONENTS_CHECKER_TYPE_MAPPING
+from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
 from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel
 from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.incremental import (
@@ -38,26 +60,50 @@ from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import Da
 from airbyte_cdk.sources.declarative.incremental.per_partition_with_global import (
     PerPartitionWithGlobalCursor,
 )
-from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedBoolean
+from airbyte_cdk.sources.declarative.models import FileUploader
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ConcurrencyLevel as ConcurrencyLevelModel,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     DatetimeBasedCursor as DatetimeBasedCursorModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    DeclarativeStream as DeclarativeStreamModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     IncrementingCountCursor as IncrementingCountCursorModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    Spec as SpecModel,
+)
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    StateDelegatingStream as StateDelegatingStreamModel,
+)
+from airbyte_cdk.sources.declarative.parsers.custom_code_compiler import (
+    get_registered_components_module,
+)
+from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import (
+    ManifestComponentTransformer,
+)
+from airbyte_cdk.sources.declarative.parsers.manifest_normalizer import (
+    ManifestNormalizer,
+)
+from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import (
+    ManifestReferenceResolver,
+)
 from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
     ModelToComponentFactory,
 )
 from airbyte_cdk.sources.declarative.partition_routers import AsyncJobPartitionRouter
+from airbyte_cdk.sources.declarative.resolvers import COMPONENTS_RESOLVER_TYPE_MAPPING
 from airbyte_cdk.sources.declarative.retrievers import AsyncRetriever, Retriever, SimpleRetriever
+from airbyte_cdk.sources.declarative.spec.spec import Spec
 from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import (
     DeclarativePartitionFactory,
     StreamSlicerPartitionGenerator,
 )
-from airbyte_cdk.sources.declarative.types import ConnectionDefinition
+from airbyte_cdk.sources.declarative.types import Config, ConnectionDefinition
 from airbyte_cdk.sources.message.concurrent_repository import ConcurrentMessageRepository
 from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.source import TState
@@ -68,6 +114,12 @@ from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, Fina
 from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream
 from airbyte_cdk.sources.streams.concurrent.partitions.types import QueueItem
+from airbyte_cdk.sources.utils.slice_logger import (
+    AlwaysLogSliceLogger,
+    DebugSliceLogger,
+    SliceLogger,
+)
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 
 @dataclass
@@ -85,8 +137,33 @@ class TestLimits:
     max_streams: int = field(default=DEFAULT_MAX_STREAMS)
 
 
-class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
-    # By default, we defer to a value of 2. A value lower than than could cause a PartitionEnqueuer to be stuck in a state of deadlock
+def _get_declarative_component_schema() -> Dict[str, Any]:
+    try:
+        raw_component_schema = pkgutil.get_data(
+            "airbyte_cdk", "sources/declarative/declarative_component_schema.yaml"
+        )
+        if raw_component_schema is not None:
+            declarative_component_schema = yaml.load(raw_component_schema, Loader=yaml.SafeLoader)
+            return declarative_component_schema  # type: ignore
+        else:
+            raise RuntimeError(
+                "Failed to read manifest component json schema required for deduplication"
+            )
+    except FileNotFoundError as e:
+        raise FileNotFoundError(
+            f"Failed to read manifest component json schema required for deduplication: {e}"
+        )
+
+
+# todo: AbstractSource can be removed once we've completely moved off all legacy synchronous CDK code paths
+#  and replaced with implementing the source.py:Source class
+#
+# todo: The `ConcurrentDeclarativeSource.message_repository()` method can also be removed once AbstractSource
+#  is no longer inherited from since the only external dependency is from that class.
+#
+# todo: It is worth investigating removal of the Generic[TState] since it will always be Optional[List[AirbyteStateMessage]]
+class ConcurrentDeclarativeSource(AbstractSource, Generic[TState]):
+    # By default, we defer to a value of 2. A value lower than could cause a PartitionEnqueuer to be stuck in a state of deadlock
     # because it has hit the limit of futures but not partition reader is consuming them.
     _LOWEST_SAFE_CONCURRENCY_LEVEL = 2
 
@@ -104,6 +181,10 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         config_path: Optional[str] = None,
         **kwargs: Any,
     ) -> None:
+        self.logger = logging.getLogger(f"airbyte.{self.name}")
+
+        self._limits = limits
+
         # todo: We could remove state from initialization. Now that streams are grouped during the read(), a source
         #  no longer needs to store the original incoming state. But maybe there's an edge case?
         self._connector_state_manager = ConnectorStateManager(state=state)  # type: ignore  # state is always in the form of List[AirbyteStateMessage]. The ConnectorStateManager should use generics, but this can be done later
@@ -132,18 +213,40 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             disable_cache=True if limits else False,
         )
 
-        self._limits = limits
+        self._should_normalize = normalize_manifest
+        self._should_migrate = migrate_manifest
+        self._declarative_component_schema = _get_declarative_component_schema()
+        # If custom components are needed, locate and/or register them.
+        self.components_module: ModuleType | None = get_registered_components_module(config=config)
+        # set additional attributes
+        self._debug = debug
+        self._emit_connector_builder_messages = emit_connector_builder_messages
+        self._constructor = (
+            component_factory
+            if component_factory
+            else ModelToComponentFactory(
+                emit_connector_builder_messages=emit_connector_builder_messages,
+                max_concurrent_async_job_count=source_config.get("max_concurrent_async_job_count"),
+            )
+        )
 
-        super().__init__(
-            source_config=source_config,
-            config=config,
-            debug=debug,
-            emit_connector_builder_messages=emit_connector_builder_messages,
-            migrate_manifest=migrate_manifest,
-            normalize_manifest=normalize_manifest,
-            component_factory=component_factory,
-            config_path=config_path,
+        self._message_repository = self._constructor.get_message_repository()
+        self._slice_logger: SliceLogger = (
+            AlwaysLogSliceLogger() if emit_connector_builder_messages else DebugSliceLogger()
+        )
+
+        # resolve all components in the manifest
+        self._source_config = self._pre_process_manifest(dict(source_config))
+        # validate resolved manifest against the declarative component schema
+        self._validate_source()
+        # apply additional post-processing to the manifest
+        self._post_process_manifest()
+
+        spec: Optional[Mapping[str, Any]] = self._source_config.get("spec")
+        self._spec_component: Optional[Spec] = (
+            self._constructor.create_component(SpecModel, spec, dict()) if spec else None
         )
+        self._config = self._migrate_and_transform_config(config_path, config) or {}
 
         concurrency_level_from_manifest = self._source_config.get("concurrency_level")
         if concurrency_level_from_manifest:
@@ -171,15 +274,141 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             logger=self.logger,
             slice_logger=self._slice_logger,
             queue=queue,
-            message_repository=self.message_repository,
+            message_repository=self._message_repository,
+        )
+
+    def _pre_process_manifest(self, manifest: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Preprocesses the provided manifest dictionary by resolving any manifest references.
+
+        This method modifies the input manifest in place, resolving references using the
+        ManifestReferenceResolver to ensure all references within the manifest are properly handled.
+
+        Args:
+            manifest (Dict[str, Any]): The manifest dictionary to preprocess and resolve references in.
+
+        Returns:
+            None
+        """
+        # For ease of use we don't require the type to be specified at the top level manifest, but it should be included during processing
+        manifest = self._fix_source_type(manifest)
+        # Resolve references in the manifest
+        resolved_manifest = ManifestReferenceResolver().preprocess_manifest(manifest)
+        # Propagate types and parameters throughout the manifest
+        propagated_manifest = ManifestComponentTransformer().propagate_types_and_parameters(
+            "", resolved_manifest, {}
         )
 
+        return propagated_manifest
+
+    def _fix_source_type(self, manifest: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Fix the source type in the manifest. This is necessary because the source type is not always set in the manifest.
+        """
+        if "type" not in manifest:
+            manifest["type"] = "DeclarativeSource"
+
+        return manifest
+
+    def _post_process_manifest(self) -> None:
+        """
+        Post-processes the manifest after validation.
+        This method is responsible for any additional modifications or transformations needed
+        after the manifest has been validated and before it is used in the source.
+        """
+        # apply manifest migration, if required
+        self._migrate_manifest()
+        # apply manifest normalization, if required
+        self._normalize_manifest()
+
+    def _migrate_manifest(self) -> None:
+        """
+        This method is used to migrate the manifest. It should be called after the manifest has been validated.
+        The migration is done in place, so the original manifest is modified.
+
+        The original manifest is returned if any error occurs during migration.
+        """
+        if self._should_migrate:
+            manifest_migrator = ManifestMigrationHandler(self._source_config)
+            self._source_config = manifest_migrator.apply_migrations()
+            # validate migrated manifest against the declarative component schema
+            self._validate_source()
+
+    def _normalize_manifest(self) -> None:
+        """
+        This method is used to normalize the manifest. It should be called after the manifest has been validated.
+
+        Connector Builder UI rendering requires the manifest to be in a specific format.
+         - references have been resolved
+         - the commonly used definitions are extracted to the `definitions.linked.*`
+        """
+        if self._should_normalize:
+            normalizer = ManifestNormalizer(self._source_config, self._declarative_component_schema)
+            self._source_config = normalizer.normalize()
+
+    def _validate_source(self) -> None:
+        """
+        Validates the connector manifest against the declarative component schema
+        """
+
+        try:
+            validate(self._source_config, self._declarative_component_schema)
+        except ValidationError as e:
+            raise ValidationError(
+                "Validation against json schema defined in declarative_component_schema.yaml schema failed"
+            ) from e
+
+    def _migrate_and_transform_config(
+        self,
+        config_path: Optional[str],
+        config: Optional[Config],
+    ) -> Optional[Config]:
+        if not config:
+            return None
+        if not self._spec_component:
+            return config
+        mutable_config = dict(config)
+        self._spec_component.migrate_config(mutable_config)
+        if mutable_config != config:
+            if config_path:
+                with open(config_path, "w") as f:
+                    json.dump(mutable_config, f)
+            control_message = create_connector_config_control_message(mutable_config)
+            print(orjson.dumps(AirbyteMessageSerializer.dump(control_message)).decode())
+        self._spec_component.transform_config(mutable_config)
+        return mutable_config
+
+    def configure(self, config: Mapping[str, Any], temp_dir: str) -> Mapping[str, Any]:
+        config = self._config or config
+        return super().configure(config, temp_dir)
+
+    @property
+    def resolved_manifest(self) -> Mapping[str, Any]:
+        """
+        Returns the resolved manifest configuration for the source.
+
+        This property provides access to the internal source configuration as a mapping,
+        which contains all settings and parameters required to define the source's behavior.
+
+        Returns:
+            Mapping[str, Any]: The resolved source configuration manifest.
+        """
+        return self._source_config
+
+    # TODO: Deprecate this class once ConcurrentDeclarativeSource no longer inherits AbstractSource
+    @property
+    def message_repository(self) -> MessageRepository:
+        return self._message_repository
+
     # TODO: Remove this. This property is necessary to safely migrate Stripe during the transition state.
     @property
     def is_partially_declarative(self) -> bool:
         """This flag used to avoid unexpected AbstractStreamFacade processing as concurrent streams."""
         return False
 
+    def deprecation_warnings(self) -> List[ConnectorBuilderLogMessage]:
+        return self._constructor.get_model_deprecations()
+
     def read(
         self,
         logger: logging.Logger,
@@ -237,7 +466,140 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
 
         In both case, we will assume that calling the DeclarativeStream is perfectly fine as the result for these is the same regardless of if it is a DeclarativeStream or a DefaultStream (concurrent). This should simply be removed once we have moved away from the mentioned code paths above.
         """
-        return super().streams(config)
+
+        if self._spec_component:
+            self._spec_component.validate_config(config)
+
+        stream_configs = (
+            self._stream_configs(self._source_config, config=config) + self.dynamic_streams
+        )
+
+        api_budget_model = self._source_config.get("api_budget")
+        if api_budget_model:
+            self._constructor.set_api_budget(api_budget_model, config)
+
+        source_streams = [
+            self._constructor.create_component(
+                (
+                    StateDelegatingStreamModel
+                    if stream_config.get("type") == StateDelegatingStreamModel.__name__
+                    else DeclarativeStreamModel
+                ),
+                stream_config,
+                config,
+                emit_connector_builder_messages=self._emit_connector_builder_messages,
+            )
+            for stream_config in self._initialize_cache_for_parent_streams(deepcopy(stream_configs))
+        ]
+        return source_streams
+
+    @staticmethod
+    def _initialize_cache_for_parent_streams(
+        stream_configs: List[Dict[str, Any]],
+    ) -> List[Dict[str, Any]]:
+        parent_streams = set()
+
+        def update_with_cache_parent_configs(
+            parent_configs: list[dict[str, Any]],
+        ) -> None:
+            for parent_config in parent_configs:
+                parent_streams.add(parent_config["stream"]["name"])
+                if parent_config["stream"]["type"] == "StateDelegatingStream":
+                    parent_config["stream"]["full_refresh_stream"]["retriever"]["requester"][
+                        "use_cache"
+                    ] = True
+                    parent_config["stream"]["incremental_stream"]["retriever"]["requester"][
+                        "use_cache"
+                    ] = True
+                else:
+                    parent_config["stream"]["retriever"]["requester"]["use_cache"] = True
+
+        for stream_config in stream_configs:
+            if stream_config.get("incremental_sync", {}).get("parent_stream"):
+                parent_streams.add(stream_config["incremental_sync"]["parent_stream"]["name"])
+                stream_config["incremental_sync"]["parent_stream"]["retriever"]["requester"][
+                    "use_cache"
+                ] = True
+
+            elif stream_config.get("retriever", {}).get("partition_router", {}):
+                partition_router = stream_config["retriever"]["partition_router"]
+
+                if isinstance(partition_router, dict) and partition_router.get(
+                    "parent_stream_configs"
+                ):
+                    update_with_cache_parent_configs(partition_router["parent_stream_configs"])
+                elif isinstance(partition_router, list):
+                    for router in partition_router:
+                        if router.get("parent_stream_configs"):
+                            update_with_cache_parent_configs(router["parent_stream_configs"])
+
+        for stream_config in stream_configs:
+            if stream_config["name"] in parent_streams:
+                if stream_config["type"] == "StateDelegatingStream":
+                    stream_config["full_refresh_stream"]["retriever"]["requester"]["use_cache"] = (
+                        True
+                    )
+                    stream_config["incremental_stream"]["retriever"]["requester"]["use_cache"] = (
+                        True
+                    )
+                else:
+                    stream_config["retriever"]["requester"]["use_cache"] = True
+        return stream_configs
+
+    def spec(self, logger: logging.Logger) -> ConnectorSpecification:
+        """
+        Returns the connector specification (spec) as defined in the Airbyte Protocol. The spec is an object describing the possible
+        configurations (e.g: username and password) which can be configured when running this connector. For low-code connectors, this
+        will first attempt to load the spec from the manifest's spec block, otherwise it will load it from "spec.yaml" or "spec.json"
+        in the project root.
+        """
+        return (
+            self._spec_component.generate_spec() if self._spec_component else super().spec(logger)
+        )
+
+    def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
+        return super().check(logger, config)
+
+    def check_connection(
+        self, logger: logging.Logger, config: Mapping[str, Any]
+    ) -> Tuple[bool, Any]:
+        """
+        :param logger: The source logger
+        :param config: The user-provided configuration as specified by the source's spec.
+          This usually contains information required to check connection e.g. tokens, secrets and keys etc.
+        :return: A tuple of (boolean, error). If boolean is true, then the connection check is successful
+          and we can connect to the underlying data source using the provided configuration.
+          Otherwise, the input config cannot be used to connect to the underlying data source,
+          and the "error" object should describe what went wrong.
+          The error object will be cast to string to display the problem to the user.
+        """
+        return self.connection_checker.check_connection(self, logger, config)
+
+    @property
+    def connection_checker(self) -> ConnectionChecker:
+        check = self._source_config["check"]
+        if "type" not in check:
+            check["type"] = "CheckStream"
+        check_stream = self._constructor.create_component(
+            COMPONENTS_CHECKER_TYPE_MAPPING[check["type"]],
+            check,
+            dict(),
+            emit_connector_builder_messages=self._emit_connector_builder_messages,
+        )
+        if isinstance(check_stream, ConnectionChecker):
+            return check_stream
+        else:
+            raise ValueError(
+                f"Expected to generate a ConnectionChecker component, but received {check_stream.__class__}"
+            )
+
+    @property
+    def dynamic_streams(self) -> List[Dict[str, Any]]:
+        return self._dynamic_stream_configs(
+            manifest=self._source_config,
+            config=self._config,
+            with_dynamic_stream_name=True,
+        )
 
     def _group_streams(
         self, config: Mapping[str, Any]
@@ -333,7 +695,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                                 stream_name=declarative_stream.name,
                                 schema_loader=declarative_stream._schema_loader,  # type: ignore  # We are accessing the private property but the public one is optional and we will remove this code soonish
                                 retriever=retriever,
-                                message_repository=self.message_repository,
+                                message_repository=self._message_repository,
                                 max_records_limit=self._limits.max_records
                                 if self._limits
                                 else None,
@@ -370,7 +732,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                                 stream_name=declarative_stream.name,
                                 schema_loader=declarative_stream._schema_loader,  # type: ignore  # We are accessing the private property but the public one is optional and we will remove this code soonish
                                 retriever=retriever,
-                                message_repository=self.message_repository,
+                                message_repository=self._message_repository,
                                 max_records_limit=self._limits.max_records
                                 if self._limits
                                 else None,
@@ -404,7 +766,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             stream_name=declarative_stream.name,
                             schema_loader=declarative_stream._schema_loader,  # type: ignore  # We are accessing the private property but the public one is optional and we will remove this code soonish
                             retriever=declarative_stream.retriever,
-                            message_repository=self.message_repository,
+                            message_repository=self._message_repository,
                             max_records_limit=self._limits.max_records if self._limits else None,
                         ),
                         declarative_stream.retriever.stream_slicer,
@@ -416,7 +778,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                     final_state_cursor = FinalStateCursor(
                         stream_name=declarative_stream.name,
                         stream_namespace=declarative_stream.namespace,
-                        message_repository=self.message_repository,
+                        message_repository=self._message_repository,
                     )
 
                     concurrent_streams.append(
@@ -468,7 +830,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             stream_name=declarative_stream.name,
                             schema_loader=declarative_stream._schema_loader,  # type: ignore  # We are accessing the private property but the public one is optional and we will remove this code soonish
                             retriever=retriever,
-                            message_repository=self.message_repository,
+                            message_repository=self._message_repository,
                             max_records_limit=self._limits.max_records if self._limits else None,
                         ),
                         perpartition_cursor,
@@ -502,6 +864,127 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
 
         return concurrent_streams, synchronous_streams
 
+    def _stream_configs(
+        self, manifest: Mapping[str, Any], config: Mapping[str, Any]
+    ) -> List[Dict[str, Any]]:
+        # This has a warning flag for static, but after we finish part 4 we'll replace manifest with self._source_config
+        stream_configs = []
+        for current_stream_config in manifest.get("streams", []):
+            if (
+                "type" in current_stream_config
+                and current_stream_config["type"] == "ConditionalStreams"
+            ):
+                interpolated_boolean = InterpolatedBoolean(
+                    condition=current_stream_config.get("condition"),
+                    parameters={},
+                )
+
+                if interpolated_boolean.eval(config=config):
+                    stream_configs.extend(current_stream_config.get("streams", []))
+            else:
+                if "type" not in current_stream_config:
+                    current_stream_config["type"] = "DeclarativeStream"
+                stream_configs.append(current_stream_config)
+        return stream_configs
+
+    def _dynamic_stream_configs(
+        self,
+        manifest: Mapping[str, Any],
+        config: Mapping[str, Any],
+        with_dynamic_stream_name: Optional[bool] = None,
+    ) -> List[Dict[str, Any]]:
+        dynamic_stream_definitions: List[Dict[str, Any]] = manifest.get("dynamic_streams", [])
+        dynamic_stream_configs: List[Dict[str, Any]] = []
+        seen_dynamic_streams: Set[str] = set()
+
+        for dynamic_definition_index, dynamic_definition in enumerate(dynamic_stream_definitions):
+            components_resolver_config = dynamic_definition["components_resolver"]
+
+            if not components_resolver_config:
+                raise ValueError(
+                    f"Missing 'components_resolver' in dynamic definition: {dynamic_definition}"
+                )
+
+            resolver_type = components_resolver_config.get("type")
+            if not resolver_type:
+                raise ValueError(
+                    f"Missing 'type' in components resolver configuration: {components_resolver_config}"
+                )
+
+            if resolver_type not in COMPONENTS_RESOLVER_TYPE_MAPPING:
+                raise ValueError(
+                    f"Invalid components resolver type '{resolver_type}'. "
+                    f"Expected one of {list(COMPONENTS_RESOLVER_TYPE_MAPPING.keys())}."
+                )
+
+            if "retriever" in components_resolver_config:
+                components_resolver_config["retriever"]["requester"]["use_cache"] = True
+
+            # Create a resolver for dynamic components based on type
+            if resolver_type == "HttpComponentsResolver":
+                components_resolver = self._constructor.create_component(
+                    model_type=COMPONENTS_RESOLVER_TYPE_MAPPING[resolver_type],
+                    component_definition=components_resolver_config,
+                    config=config,
+                    stream_name=dynamic_definition.get("name"),
+                )
+            else:
+                components_resolver = self._constructor.create_component(
+                    model_type=COMPONENTS_RESOLVER_TYPE_MAPPING[resolver_type],
+                    component_definition=components_resolver_config,
+                    config=config,
+                )
+
+            stream_template_config = dynamic_definition["stream_template"]
+
+            for dynamic_stream in components_resolver.resolve_components(
+                stream_template_config=stream_template_config
+            ):
+                # Get the use_parent_parameters configuration from the dynamic definition
+                # Default to True for backward compatibility, since connectors were already using it by default when this param was added
+                use_parent_parameters = dynamic_definition.get("use_parent_parameters", True)
+
+                dynamic_stream = {
+                    **ManifestComponentTransformer().propagate_types_and_parameters(
+                        "", dynamic_stream, {}, use_parent_parameters=use_parent_parameters
+                    )
+                }
+
+                if "type" not in dynamic_stream:
+                    dynamic_stream["type"] = "DeclarativeStream"
+
+                # Ensure that each stream is created with a unique name
+                name = dynamic_stream.get("name")
+
+                if with_dynamic_stream_name:
+                    dynamic_stream["dynamic_stream_name"] = dynamic_definition.get(
+                        "name", f"dynamic_stream_{dynamic_definition_index}"
+                    )
+
+                if not isinstance(name, str):
+                    raise ValueError(
+                        f"Expected stream name {name} to be a string, got {type(name)}."
+                    )
+
+                if name in seen_dynamic_streams:
+                    error_message = f"Dynamic streams list contains a duplicate name: {name}. Please contact Airbyte Support."
+                    failure_type = FailureType.system_error
+
+                    if resolver_type == "ConfigComponentsResolver":
+                        error_message = f"Dynamic streams list contains a duplicate name: {name}. Please check your configuration."
+                        failure_type = FailureType.config_error
+
+                    raise AirbyteTracedException(
+                        message=error_message,
+                        internal_message=error_message,
+                        failure_type=failure_type,
+                    )
+
+                seen_dynamic_streams.add(name)
+                dynamic_stream_configs.append(dynamic_stream)
+
+        return dynamic_stream_configs
+
     def _is_concurrent_cursor_incremental_without_partition_routing(
         self,
         declarative_stream: DeclarativeStream,