airbyte-cdk 6.52.0.dev0__py3-none-any.whl → 6.52.0.dev1__py3-none-any.whl

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -4055,6 +4055,9 @@ definitions:
         title: Value Type
         description: The expected data type of the value. If omitted, the type will be inferred from the value provided.
         "$ref": "#/definitions/ValueType"
+      create_or_update:
+        type: boolean
+        default: false
       $parameters:
         type: object
         additionalProperties: true
@@ -4106,6 +4109,12 @@ definitions:
           - ["data"]
           - ["data", "streams"]
           - ["data", "{{ parameters.name }}"]
+      default_values:
+        title: Default Values
+        description: A list of default values, each matching the structure expected from the parsed component value.
+        type: array
+        items:
+          type: object
       $parameters:
         type: object
         additionalProperties: true
@@ -4117,7 +4126,11 @@ definitions:
         type: string
         enum: [ConfigComponentsResolver]
       stream_config:
-        "$ref": "#/definitions/StreamConfig"
+        anyOf:
+          - type: array
+            items:
+              "$ref": "#/definitions/StreamConfig"
+          - "$ref": "#/definitions/StreamConfig"
       components_mapping:
         type: array
         items:

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -300,9 +300,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
             }
         )
 
-        stream_configs = self._stream_configs(self._source_config) + self._dynamic_stream_configs(
-            self._source_config, config
-        )
+        stream_configs = self._stream_configs(self._source_config) + self.dynamic_streams
 
         api_budget_model = self._source_config.get("api_budget")
         if api_budget_model:

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1478,6 +1478,7 @@ class ComponentMappingDefinition(BaseModel):
         description="The expected data type of the value. If omitted, the type will be inferred from the value provided.",
         title="Value Type",
     )
+    create_or_update: Optional[bool] = False
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
@@ -1489,12 +1490,17 @@ class StreamConfig(BaseModel):
         examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]],
         title="Configs Pointer",
     )
+    default_values: Optional[List[Dict[str, Any]]] = Field(
+        None,
+        description="A list of default values, each matching the structure expected from the parsed component value.",
+        title="Default Values",
+    )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
 class ConfigComponentsResolver(BaseModel):
     type: Literal["ConfigComponentsResolver"]
-    stream_config: StreamConfig
+    stream_config: Union[List[StreamConfig], StreamConfig]
     components_mapping: List[ComponentMappingDefinition]
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -3754,6 +3754,7 @@ class ModelToComponentFactory:
             field_path=field_path,  # type: ignore[arg-type] # field_path can be str and InterpolatedString
             value=interpolated_value,
             value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type),
+            create_or_update=model.create_or_update,
             parameters=model.parameters or {},
         )
 
@@ -3800,16 +3801,24 @@ class ModelToComponentFactory:
 
         return StreamConfig(
             configs_pointer=model_configs_pointer,
+            default_values=model.default_values,
             parameters=model.parameters or {},
         )
 
     def create_config_components_resolver(
         self, model: ConfigComponentsResolverModel, config: Config
     ) -> Any:
-        stream_config = self._create_component_from_model(
-            model.stream_config, config=config, parameters=model.parameters or {}
+        model_stream_configs = (
+            model.stream_config if isinstance(model.stream_config, list) else [model.stream_config]
         )
 
+        stream_configs = [
+            self._create_component_from_model(
+                stream_config, config=config, parameters=model.parameters or {}
+            )
+            for stream_config in model_stream_configs
+        ]
+
         components_mapping = [
             self._create_component_from_model(
                 model=components_mapping_definition_model,
@@ -3822,7 +3831,7 @@ class ModelToComponentFactory:
         ]
 
         return ConfigComponentsResolver(
-            stream_config=stream_config,
+            stream_configs=stream_configs,
             config=config,
             components_mapping=components_mapping,
             parameters=model.parameters or {},

airbyte_cdk/sources/declarative/resolvers/components_resolver.py

@@ -22,6 +22,7 @@ class ComponentMappingDefinition:
     value: Union["InterpolatedString", str]
     value_type: Optional[Type[Any]]
     parameters: InitVar[Mapping[str, Any]]
+    create_or_update: Optional[bool] = False
 
 
 @dataclass(frozen=True)
@@ -34,6 +35,7 @@ class ResolvedComponentMappingDefinition:
     value: "InterpolatedString"
     value_type: Optional[Type[Any]]
     parameters: InitVar[Mapping[str, Any]]
+    create_or_update: Optional[bool] = False
 
 
 @deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)

airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py

@@ -4,10 +4,13 @@
 
 from copy import deepcopy
 from dataclasses import InitVar, dataclass, field
-from typing import Any, Dict, Iterable, List, Mapping, Union
+from itertools import product
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union
 
 import dpath
+import yaml
 from typing_extensions import deprecated
+from yaml.parser import ParserError
 
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
@@ -28,6 +31,7 @@ class StreamConfig:
 
     configs_pointer: List[Union[InterpolatedString, str]]
     parameters: InitVar[Mapping[str, Any]]
+    default_values: Optional[List[Any]] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self.configs_pointer = [
@@ -48,7 +52,7 @@ class ConfigComponentsResolver(ComponentsResolver):
         parameters (InitVar[Mapping[str, Any]]): Additional parameters for interpolation.
     """
 
-    stream_config: StreamConfig
+    stream_configs: List[StreamConfig]
     config: Config
     components_mapping: List[ComponentMappingDefinition]
     parameters: InitVar[Mapping[str, Any]]
@@ -82,6 +86,7 @@ class ConfigComponentsResolver(ComponentsResolver):
                         field_path=field_path,
                         value=interpolated_value,
                         value_type=component_mapping.value_type,
+                        create_or_update=component_mapping.create_or_update,
                         parameters=parameters,
                     )
                 )
@@ -90,18 +95,45 @@ class ConfigComponentsResolver(ComponentsResolver):
                     f"Expected a string or InterpolatedString for value in mapping: {component_mapping}"
                 )
 
+    @staticmethod
+    def _merge_combination(combo: Iterable[Tuple[int, Any]]) -> Dict[str, Any]:
+        """Collapse a combination of ``(idx, elem)`` into one config dict."""
+        result: Dict[str, Any] = {}
+        for config_index, (elem_index, elem) in enumerate(combo):
+            if isinstance(elem, dict):
+                result.update(elem)
+            else:
+                # keep non-dict values under an artificial name
+                result.setdefault(f"source_config_{config_index}", (elem_index, elem))
+        return result
+
     @property
-    def _stream_config(self) -> Iterable[Mapping[str, Any]]:
-        path = [
-            node.eval(self.config) if not isinstance(node, str) else node
-            for node in self.stream_config.configs_pointer
+    def _stream_config(self) -> List[Dict[str, Any]]:
+        """
+        Build every unique stream-configuration combination defined by
+        each ``StreamConfig`` and any ``default_values``.
+        """
+        all_indexed_streams = []
+        for stream_config in self.stream_configs:
+            path = [
+                node.eval(self.config) if not isinstance(node, str) else node
+                for node in stream_config.configs_pointer
+            ]
+            stream_configs_raw = dpath.get(dict(self.config), path, default=[])
+            stream_configs = (
+                list(stream_configs_raw)
+                if isinstance(stream_configs_raw, list)
+                else [stream_configs_raw]
+            )
+
+            if stream_config.default_values:
+                stream_configs.extend(stream_config.default_values)
+
+            all_indexed_streams.append([(i, item) for i, item in enumerate(stream_configs)])
+        return [
+            self._merge_combination(combo)  # type: ignore[arg-type]
+            for combo in product(*all_indexed_streams)
         ]
-        stream_config = dpath.get(dict(self.config), path, default=[])
-
-        if not isinstance(stream_config, list):
-            stream_config = [stream_config]
-
-        return stream_config
 
     def resolve_components(
         self, stream_template_config: Dict[str, Any]
@@ -130,7 +162,27 @@ class ConfigComponentsResolver(ComponentsResolver):
                 )
 
                 path = [path.eval(self.config, **kwargs) for path in resolved_component.field_path]
+                parsed_value = self._parse_yaml_if_possible(value)
+                updated = dpath.set(updated_config, path, parsed_value)
 
-                dpath.set(updated_config, path, value)
+                if parsed_value and not updated and resolved_component.create_or_update:
+                    dpath.new(updated_config, path, parsed_value)
 
             yield updated_config
+
+    @staticmethod
+    def _parse_yaml_if_possible(value: Any) -> Any:
+        """
+        Try to turn value into a Python object by YAML-parsing it.
+
+        * If value is a `str` and can be parsed by `yaml.safe_load`,
+          return the parsed result.
+        * If parsing fails (`yaml.parser.ParserError`) – or value is not
+          a string at all – return the original value unchanged.
+        """
+        if isinstance(value, str):
+            try:
+                return yaml.safe_load(value)
+            except ParserError:  # "{{ record[0] in ['cohortActiveUsers'] }}"   # not valid YAML
+                return value
+        return value

{airbyte_cdk-6.52.0.dev0.dist-info → airbyte_cdk-6.52.0.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.52.0.dev0
+Version: 6.52.0.dev1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.52.0.dev0.dist-info → airbyte_cdk-6.52.0.dev1.dist-info}/RECORD

@@ -89,7 +89,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=OKor1mDD
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=ttq2s4aivIbvIj2OWy0-CNb2cVXYVTzOgXS9PLbMNWU,176666
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=LklBDA_hfKidQKWp4y8NCYBx4HkBVFAxhoTexBB2AU4,177075
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=qmyMnnet92eGc3C22yBtpvD5UZjqdhsAafP_zxI5wp8,1814
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
@@ -127,20 +127,20 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkH
 airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
 airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
 airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=UYSJ5gW7TkHALYnNvUnRP3RlyGwGuRMObF3BHuNzjJM,5320
-airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=AVb86oWIAxAOrE3KAyNr81bF5Dd2QjKdQ6E4Jl4yrQw,24719
+airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=aS_YxqUWgxyfyzwAZpV736RFg-m5a0sn-V7PPMBGVr4,24660
 airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=V2lpYE9LJKvz6BUViHk4vaRGndxNABmPbDCtyYdkqaE,4013
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
 airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py,sha256=Imnj3yef0aqRdLfaUxkIYISUb8YkiPrRH_wBd-x8HjM,5999
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=YhQl_vjQyWl1GJTuPyhg1aKWQazbqHROZ5v0dIg7auE,125123
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=ooVQMvPd4Dd-0XHjOXUFHdbSMe4PaP9-UjQ23ulRxz4,125426
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9Rbu5OexXSDN9QWDo8YU4tT9M2LDVOgGA,802
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=2UdpCz3yi7ISZTyqkQXSSy3dMxeyOWqV7OlAS5b9GVg,11568
 airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=j7MUNaWDpsxe5yDzE3-id_wivjNrSjsJa1VgE4oSRlo,174401
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=7-q8icBPnEHbPa0vonz5KjsalYrrqJasIcLXK1TITlc,174730
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -194,8 +194,8 @@ airbyte_cdk/sources/declarative/requesters/request_options/request_options_provi
 airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
 airbyte_cdk/sources/declarative/requesters/requester.py,sha256=T6tMx_Bx4iT-0YVjY7IzgRil-gaIu9n01b1iwpTh3Ek,5516
 airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=NiDcz5qi8HPsfX94MUmnX0Rgs_kQXGvucOmJjNWlxKQ,1207
-airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb9artL4ZkeqN8RmEykHH6FJgqXD7fCEnh1X0,1936
-airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
+airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=oJIpy66ep8n-QOc8GwpllApTRcl4QtQhkTw5fWWra2w,2026
+airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=sD3N7nmqDjLsau8P2DE7DYOHdFTYjC_2nIB6454BNYk,7556
 airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=nQepwG_RfW53sgwvK5dLPqfCx0VjsQ83nYoPjBMAaLM,527
 airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=6oZtnCHm9NdDvjTSrVwPQOXGSdETSIR7eWH2vFjM7jI,4855
@@ -419,9 +419,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.52.0.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.52.0.dev0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.52.0.dev0.dist-info/METADATA,sha256=EWPjAFT5JFlV51VuHFHIciFphjfoYNyJ9prFTbWkbyM,6348
-airbyte_cdk-6.52.0.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.52.0.dev0.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.52.0.dev0.dist-info/RECORD,,
+airbyte_cdk-6.52.0.dev1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.52.0.dev1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.52.0.dev1.dist-info/METADATA,sha256=Atlb-SeF8vrGZIBbNcskhXmEpQBqFgfrFjM41G23cps,6348
+airbyte_cdk-6.52.0.dev1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.52.0.dev1.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.52.0.dev1.dist-info/RECORD,,