airbyte-cdk 6.55.1__py3-none-any.whl → 6.56.0__py3-none-any.whl

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -4186,6 +4186,46 @@ definitions:
       - type
       - stream_config
       - components_mapping
+  StreamParametersDefinition:
+    title: Stream Parameters Definition
+    description: (This component is experimental. Use at your own risk.) Represents a stream parameters definition to set up dynamic streams from defined values in manifest.
+    type: object
+    required:
+      - type
+      - list_of_parameters_for_stream
+    properties:
+      type:
+        type: string
+        enum: [StreamParametersDefinition]
+      list_of_parameters_for_stream:
+        title: Stream Parameters
+        description: A list of object of parameters for stream, each object in the list represents params for one stream.
+        type: array
+        items:
+          type: object
+        examples:
+          - [{"name": "test stream", "$parameters": {"entity": "test entity"}, "primary_key": "test key"}]
+  ParametrizedComponentsResolver:
+    type: object
+    title: Parametrized Components Resolver
+    description: (This component is experimental. Use at your own risk.) Resolves and populates dynamic streams from defined parametrized values in manifest.
+    properties:
+      type:
+        type: string
+        enum: [ParametrizedComponentsResolver]
+      stream_parameters:
+        "$ref": "#/definitions/StreamParametersDefinition"
+      components_mapping:
+        type: array
+        items:
+          "$ref": "#/definitions/ComponentMappingDefinition"
+      $parameters:
+        type: object
+        additionalProperties: true
+    required:
+      - type
+      - stream_parameters
+      - components_mapping
   DynamicDeclarativeStream:
     type: object
     description: (This component is experimental. Use at your own risk.) A component that described how will be created declarative streams based on stream template.
@@ -4212,6 +4252,7 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/HttpComponentsResolver"
           - "$ref": "#/definitions/ConfigComponentsResolver"
+          - "$ref": "#/definitions/ParametrizedComponentsResolver"
     required:
       - type
       - stream_template

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1512,6 +1512,31 @@ class ConfigComponentsResolver(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class StreamParametersDefinition(BaseModel):
+    type: Literal["StreamParametersDefinition"]
+    list_of_parameters_for_stream: List[Dict[str, Any]] = Field(
+        ...,
+        description="A list of object of parameters for stream, each object in the list represents params for one stream.",
+        examples=[
+            [
+                {
+                    "name": "test stream",
+                    "$parameters": {"entity": "test entity"},
+                    "primary_key": "test key",
+                }
+            ]
+        ],
+        title="Stream Parameters",
+    )
+
+
+class ParametrizedComponentsResolver(BaseModel):
+    type: Literal["ParametrizedComponentsResolver"]
+    stream_parameters: StreamParametersDefinition
+    components_mapping: List[ComponentMappingDefinition]
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class RequestBodyPlainText(BaseModel):
     type: Literal["RequestBodyPlainText"]
     value: str
@@ -2943,7 +2968,9 @@ class DynamicDeclarativeStream(BaseModel):
     stream_template: Union[DeclarativeStream, StateDelegatingStream] = Field(
         ..., description="Reference to the stream template.", title="Stream Template"
     )
-    components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field(
+    components_resolver: Union[
+        HttpComponentsResolver, ConfigComponentsResolver, ParametrizedComponentsResolver
+    ] = Field(
         ...,
         description="Component resolve and populates stream templates with components values.",
         title="Components Resolver",

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -351,6 +351,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     PageIncrement as PageIncrementModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    ParametrizedComponentsResolver as ParametrizedComponentsResolverModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ParentStreamConfig as ParentStreamConfigModel,
 )
@@ -504,7 +507,9 @@ from airbyte_cdk.sources.declarative.resolvers import (
     ComponentMappingDefinition,
     ConfigComponentsResolver,
     HttpComponentsResolver,
+    ParametrizedComponentsResolver,
     StreamConfig,
+    StreamParametersDefinition,
 )
 from airbyte_cdk.sources.declarative.retrievers import (
     AsyncRetriever,
@@ -738,6 +743,7 @@ class ModelToComponentFactory:
             AsyncRetrieverModel: self.create_async_retriever,
             HttpComponentsResolverModel: self.create_http_components_resolver,
             ConfigComponentsResolverModel: self.create_config_components_resolver,
+            ParametrizedComponentsResolverModel: self.create_parametrized_components_resolver,
             StreamConfigModel: self.create_stream_config,
             ComponentMappingDefinitionModel: self.create_components_mapping_definition,
             ZipfileDecoderModel: self.create_zipfile_decoder,
@@ -3861,6 +3867,29 @@ class ModelToComponentFactory:
             parameters=model.parameters or {},
         )
 
+    def create_parametrized_components_resolver(
+        self, model: ParametrizedComponentsResolverModel, config: Config
+    ) -> ParametrizedComponentsResolver:
+        stream_parameters = StreamParametersDefinition(
+            list_of_parameters_for_stream=model.stream_parameters.list_of_parameters_for_stream
+        )
+        components_mapping = [
+            self._create_component_from_model(
+                model=components_mapping_definition_model,
+                value_type=ModelToComponentFactory._json_schema_type_name_to_type(
+                    components_mapping_definition_model.value_type
+                ),
+                config=config,
+            )
+            for components_mapping_definition_model in model.components_mapping
+        ]
+        return ParametrizedComponentsResolver(
+            stream_parameters=stream_parameters,
+            config=config,
+            components_mapping=components_mapping,
+            parameters=model.parameters or {},
+        )
+
     _UNSUPPORTED_DECODER_ERROR = (
         "Specified decoder of {decoder_type} is not supported for pagination."
         "Please set as `JsonDecoder`, `XmlDecoder`, or a `CompositeRawDecoder` with an inner_parser of `JsonParser` or `GzipParser` instead."

airbyte_cdk/sources/declarative/resolvers/__init__.py

@@ -12,6 +12,9 @@ from airbyte_cdk.sources.declarative.models import (
 from airbyte_cdk.sources.declarative.models import (
     HttpComponentsResolver as HttpComponentsResolverModel,
 )
+from airbyte_cdk.sources.declarative.models import (
+    ParametrizedComponentsResolver as ParametrizedComponentsResolverModel,
+)
 from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
     ComponentMappingDefinition,
     ComponentsResolver,
@@ -24,10 +27,15 @@ from airbyte_cdk.sources.declarative.resolvers.config_components_resolver import
 from airbyte_cdk.sources.declarative.resolvers.http_components_resolver import (
     HttpComponentsResolver,
 )
+from airbyte_cdk.sources.declarative.resolvers.parametrized_components_resolver import (
+    ParametrizedComponentsResolver,
+    StreamParametersDefinition,
+)
 
 COMPONENTS_RESOLVER_TYPE_MAPPING: Mapping[str, type[BaseModel]] = {
     "HttpComponentsResolver": HttpComponentsResolverModel,
     "ConfigComponentsResolver": ConfigComponentsResolverModel,
+    "ParametrizedComponentsResolver": ParametrizedComponentsResolverModel,
 }
 
 __all__ = [
@@ -38,4 +46,6 @@ __all__ = [
     "StreamConfig",
     "ConfigComponentsResolver",
     "COMPONENTS_RESOLVER_TYPE_MAPPING",
+    "ParametrizedComponentsResolver",
+    "StreamParametersDefinition",
 ]

airbyte_cdk/sources/declarative/resolvers/parametrized_components_resolver.py

@@ -0,0 +1,125 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from copy import deepcopy
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Dict, Iterable, List, Mapping
+
+import dpath
+import yaml
+from typing_extensions import deprecated
+from yaml.parser import ParserError
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
+    ComponentMappingDefinition,
+    ComponentsResolver,
+    ResolvedComponentMappingDefinition,
+)
+from airbyte_cdk.sources.source import ExperimentalClassWarning
+from airbyte_cdk.sources.types import Config
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class StreamParametersDefinition:
+    """
+    Represents a stream parameters definition to set up dynamic streams from defined values in manifest.
+    """
+
+    list_of_parameters_for_stream: List[Dict[str, Any]]
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class ParametrizedComponentsResolver(ComponentsResolver):
+    """
+    Resolves and populates dynamic streams from defined parametrized values in manifest.
+    """
+
+    stream_parameters: StreamParametersDefinition
+    config: Config
+    components_mapping: List[ComponentMappingDefinition]
+    parameters: InitVar[Mapping[str, Any]]
+    _resolved_components: List[ResolvedComponentMappingDefinition] = field(
+        init=False, repr=False, default_factory=list
+    )
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        """
+        Initializes and parses component mappings, converting them to resolved definitions.
+
+        Args:
+            parameters (Mapping[str, Any]): Parameters for interpolation.
+        """
+
+        for component_mapping in self.components_mapping:
+            if isinstance(component_mapping.value, (str, InterpolatedString)):
+                interpolated_value = (
+                    InterpolatedString.create(component_mapping.value, parameters=parameters)
+                    if isinstance(component_mapping.value, str)
+                    else component_mapping.value
+                )
+
+                field_path = [
+                    InterpolatedString.create(path, parameters=parameters)
+                    for path in component_mapping.field_path
+                ]
+
+                self._resolved_components.append(
+                    ResolvedComponentMappingDefinition(
+                        field_path=field_path,
+                        value=interpolated_value,
+                        value_type=component_mapping.value_type,
+                        create_or_update=component_mapping.create_or_update,
+                        parameters=parameters,
+                    )
+                )
+            else:
+                raise ValueError(
+                    f"Expected a string or InterpolatedString for value in mapping: {component_mapping}"
+                )
+
+    def resolve_components(
+        self, stream_template_config: Dict[str, Any]
+    ) -> Iterable[Dict[str, Any]]:
+        kwargs = {"stream_template_config": stream_template_config}
+
+        for components_values in self.stream_parameters.list_of_parameters_for_stream:
+            updated_config = deepcopy(stream_template_config)
+            kwargs["components_values"] = components_values  # type: ignore[assignment] # component_values will always be of type Mapping[str, Any]
+            for resolved_component in self._resolved_components:
+                valid_types = (
+                    (resolved_component.value_type,) if resolved_component.value_type else None
+                )
+                value = resolved_component.value.eval(
+                    self.config, valid_types=valid_types, **kwargs
+                )
+                path = [path.eval(self.config, **kwargs) for path in resolved_component.field_path]
+                parsed_value = self._parse_yaml_if_possible(value)
+                # https://github.com/dpath-maintainers/dpath-python/blob/master/dpath/__init__.py#L136
+                # dpath.set returns the number of changed elements, 0 when no elements changed
+                updated = dpath.set(updated_config, path, parsed_value)
+
+                if parsed_value and not updated and resolved_component.create_or_update:
+                    dpath.new(updated_config, path, parsed_value)
+
+            yield updated_config
+
+    @staticmethod
+    def _parse_yaml_if_possible(value: Any) -> Any:
+        """
+        Try to turn value into a Python object by YAML-parsing it.
+
+        * If value is a `str` and can be parsed by `yaml.safe_load`,
+          return the parsed result.
+        * If parsing fails (`yaml.parser.ParserError`) – or value is not
+          a string at all – return the original value unchanged.
+        """
+        if isinstance(value, str):
+            try:
+                return yaml.safe_load(value)
+            except ParserError:  # "{{ record[0] in ['cohortActiveUsers'] }}"   # not valid YAML
+                return value
+        return value

{airbyte_cdk-6.55.1.dist-info → airbyte_cdk-6.56.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.55.1
+Version: 6.56.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.55.1.dist-info → airbyte_cdk-6.56.0.dist-info}/RECORD

@@ -89,7 +89,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=rQz9gXp3
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=ES1gwuWHAXC9z9Dk8i0kNGK-2omTtjexiNp0iwV54tE,178751
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=JwWxM_QAIC7jAOYQMpo9fWa1aULtGhMn9Tg0bntoM58,180321
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=qmyMnnet92eGc3C22yBtpvD5UZjqdhsAafP_zxI5wp8,1814
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
@@ -133,14 +133,14 @@ airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migrati
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
 airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py,sha256=Imnj3yef0aqRdLfaUxkIYISUb8YkiPrRH_wBd-x8HjM,5999
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=xfmKKdsnPeP9hV_cjBfZEGMxdU0HJDuMwoQGrmTPYMo,126543
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=jipvV9xV_BB4-IGjdUolbY3IoaB0VPubWoCtnr1Ick0,127459
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9Rbu5OexXSDN9QWDo8YU4tT9M2LDVOgGA,802
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=2UdpCz3yi7ISZTyqkQXSSy3dMxeyOWqV7OlAS5b9GVg,11568
 airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=IaVbbuJv7DvheUaW3iBSv1qChpU8Vm1k7ZF_uESZYFU,175315
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Sr8LrlLl61eQTsJ5b8eWt3ESCPDEpJRd6DXS4P53hVE,176635
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -193,10 +193,11 @@ airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_
 airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py,sha256=8YRiDzjYvqJ-aMmKFcjqzv_-e8OZ5QG_TbpZ-nuCu6s,2590
 airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
 airbyte_cdk/sources/declarative/requesters/requester.py,sha256=T6tMx_Bx4iT-0YVjY7IzgRil-gaIu9n01b1iwpTh3Ek,5516
-airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=NiDcz5qi8HPsfX94MUmnX0Rgs_kQXGvucOmJjNWlxKQ,1207
+airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=xVhOWLQW0wFBTAtRYu3GdFebPqKCDSt1uoP2TiBGrvs,1643
 airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=oJIpy66ep8n-QOc8GwpllApTRcl4QtQhkTw5fWWra2w,2026
 airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=sD3N7nmqDjLsau8P2DE7DYOHdFTYjC_2nIB6454BNYk,7556
 airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
+airbyte_cdk/sources/declarative/resolvers/parametrized_components_resolver.py,sha256=BUmvbsMeIGusZSCd80NiTFcwcosq-uhXHGNheAFs-10,5147
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=LQQspOQS9oyOx9cGnRIz1mq-8DZCBysyDJDPqjy1HvM,449
 airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=6oZtnCHm9NdDvjTSrVwPQOXGSdETSIR7eWH2vFjM7jI,4855
 airbyte_cdk/sources/declarative/retrievers/file_uploader/__init__.py,sha256=ovs0oZGPEEw-V4asqr0D1Xty2u7tR8ADhRuHNgXGuWQ,490
@@ -422,9 +423,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.55.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.55.1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.55.1.dist-info/METADATA,sha256=l7jKuCXU7nwels800F-i0GX3S85Fjw31cHtoKadLd7Y,6388
-airbyte_cdk-6.55.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.55.1.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.55.1.dist-info/RECORD,,
+airbyte_cdk-6.56.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.56.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.56.0.dist-info/METADATA,sha256=4S60a_h2kt3cWcAJ58pJNfcC19bEBNYrIohASoS0Lio,6388
+airbyte_cdk-6.56.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.56.0.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.56.0.dist-info/RECORD,,