airbyte-cdk 6.48.15__py3-none-any.whl → 6.48.17.dev0__py3-none-any.whl

airbyte_cdk/cli/airbyte_cdk/_connector.py

@@ -39,6 +39,7 @@ poetry run airbyte-cdk connector --help
 """
 
 import os
+import sys
 from pathlib import Path
 from types import ModuleType
 
@@ -166,10 +167,11 @@ def test(
     click.echo(f"Collect only: {collect_only}")
     click.echo(f"Pytest args: {pytest_args}")
     click.echo("Invoking Pytest...")
-    pytest.main(
+    exit_code = pytest.main(
         pytest_args,
         plugins=[],
     )
+    sys.exit(exit_code)
 
 
 __all__ = [

airbyte_cdk/cli/airbyte_cdk/_secrets.py

@@ -49,7 +49,12 @@ from airbyte_cdk.utils.connector_paths import (
     resolve_connector_name_and_directory,
 )
 
-AIRBYTE_INTERNAL_GCP_PROJECT = "dataline-integration-testing"
+GCP_PROJECT_ID: str = os.environ.get("GCP_PROJECT_ID", "") or "dataline-integration-testing"
+# We put the `or` outside the `get()` because we want the `GCP_PROJECT_ID`
+# env var to be ignored if it contains an empty string, such as in CI where the
+# workflow might set it to a value that is itself actually missing or unset.
+"""The GCP project ID to use for fetching integration test secrets."""
+
 CONNECTOR_LABEL = "connector"
 GLOBAL_MASK_KEYS_URL = "https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml"
 
@@ -83,8 +88,11 @@ def secrets_cli_group() -> None:
 @click.option(
     "--gcp-project-id",
     type=str,
-    default=AIRBYTE_INTERNAL_GCP_PROJECT,
-    help=f"GCP project ID. Defaults to '{AIRBYTE_INTERNAL_GCP_PROJECT}'.",
+    default=GCP_PROJECT_ID,
+    help=(
+        "GCP project ID for retrieving integration tests credentials. "
+        "Defaults to the value of the `GCP_PROJECT_ID` environment variable, if set."
+    ),
 )
 @click.option(
     "--print-ci-secrets-masks",
@@ -95,7 +103,7 @@ def secrets_cli_group() -> None:
 )
 def fetch(
     connector: str | Path | None = None,
-    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+    gcp_project_id: str = GCP_PROJECT_ID,
     print_ci_secrets_masks: bool = False,
 ) -> None:
     """Fetch secrets for a connector from Google Secret Manager.
@@ -192,41 +200,41 @@ def fetch(
 
 
 @secrets_cli_group.command("list")
-@click.option(
-    "--connector-name",
+@click.argument(
+    "connector",
+    required=False,
     type=str,
-    help="Name of the connector to fetch secrets for. Ignored if --connector-directory is provided.",
-)
-@click.option(
-    "--connector-directory",
-    type=click.Path(exists=True, file_okay=False, path_type=Path),
-    help="Path to the connector directory.",
+    metavar="[CONNECTOR]",
 )
 @click.option(
     "--gcp-project-id",
     type=str,
-    default=AIRBYTE_INTERNAL_GCP_PROJECT,
-    help=f"GCP project ID. Defaults to '{AIRBYTE_INTERNAL_GCP_PROJECT}'.",
+    default=GCP_PROJECT_ID,
+    help=(
+        "GCP project ID for retrieving integration tests credentials. "
+        "Defaults to the value of the `GCP_PROJECT_ID` environment variable, if set."
+    ),
 )
 def list_(
-    connector_name: str | None = None,
-    connector_directory: Path | None = None,
-    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+    connector: str | Path | None = None,
+    *,
+    gcp_project_id: str = GCP_PROJECT_ID,
 ) -> None:
     """List secrets for a connector from Google Secret Manager.
 
     This command fetches secrets for a connector from Google Secret Manager and prints
     them as a table.
 
+    [CONNECTOR] can be a connector name (e.g. 'source-pokeapi'), a path to a connector directory, or omitted to use the current working directory.
+    If a string containing '/' is provided, it is treated as a path. Otherwise, it is treated as a connector name.
+
     If no connector name or directory is provided, we will look within the current working
     directory. If the current working directory is not a connector directory (e.g. starting
     with 'source-') and no connector name or path is provided, the process will fail.
     """
     click.echo("Scanning secrets...", err=True)
 
-    connector_name = connector_name or resolve_connector_name(
-        connector_directory=connector_directory or Path().resolve().absolute(),
-    )
+    connector_name, _ = resolve_connector_name_and_directory(connector)
     secrets: list[Secret] = _fetch_secret_handles(  # type: ignore
         connector_name=connector_name,
         gcp_project_id=gcp_project_id,
@@ -303,7 +311,7 @@ def _get_secret_url(secret_name: str, gcp_project_id: str) -> str:
 
 def _fetch_secret_handles(
     connector_name: str,
-    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+    gcp_project_id: str = GCP_PROJECT_ID,
 ) -> list["Secret"]:  # type: ignore
     """Fetch secrets from Google Secret Manager."""
     if not secretmanager:

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1741,6 +1741,54 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  KeyValueExtractor:
+    title: Key Value Extractor
+    description: Record extractor that extract with .
+    type: object
+    required:
+      - type
+      - keys_extractor
+      - values_extractor
+    properties:
+      type:
+        type: string
+        enum: [ KeyValueExtractor ]
+      keys_extractor:
+        description: placeholder
+        anyOf:
+          - "$ref": "#/definitions/DpathExtractor"
+          - "$ref": "#/definitions/CustomRecordExtractor"
+      values_extractor:
+        description: placeholder
+        anyOf:
+          - "$ref": "#/definitions/DpathExtractor"
+          - "$ref": "#/definitions/CustomRecordExtractor"
+      $parameters:
+        type: object
+        additionalProperties: true
+  CombinedExtractor:
+    title: Combined Extractor
+    description: Record extractor that extract with .
+    type: object
+    required:
+      - type
+      - extractors
+    properties:
+      type:
+        type: string
+        enum: [ CombinedExtractor ]
+      extractors:
+        description: placeholder
+        type: array
+        items:
+          anyOf:
+            - "$ref": "#/definitions/DpathExtractor"
+            - "$ref": "#/definitions/CombinedExtractor"
+            - "$ref": "#/definitions/KeyValueExtractor"
+            - "$ref": "#/definitions/CustomRecordExtractor"
+      $parameters:
+        type: object
+        additionalProperties: true
   DpathExtractor:
     title: Dpath Extractor
     description: Record extractor that searches a decoded response over a path defined as an array of fields.
@@ -2318,6 +2366,12 @@ definitions:
           - "$ref": "#/definitions/AsyncRetriever"
           - "$ref": "#/definitions/CustomRetriever"
           - "$ref": "#/definitions/SimpleRetriever"
+      schema_filter:
+        title: Schema Filter
+        description: placeholder
+        anyOf:
+          - "$ref": "#/definitions/RecordFilter"
+          - "$ref": "#/definitions/CustomRecordFilter"
       schema_transformations:
         title: Schema Transformations
         description: A list of transformations to be applied to the schema.
@@ -3315,6 +3369,8 @@ definitions:
       extractor:
         anyOf:
           - "$ref": "#/definitions/DpathExtractor"
+          - "$ref": "#/definitions/CombinedExtractor"
+          - "$ref": "#/definitions/KeyValueExtractor"
           - "$ref": "#/definitions/CustomRecordExtractor"
       record_filter:
         title: Record Filter
@@ -3994,6 +4050,9 @@ definitions:
         title: Value Type
         description: The expected data type of the value. If omitted, the type will be inferred from the value provided.
         "$ref": "#/definitions/ValueType"
+      create_or_update:
+        type: boolean
+        default: false
       $parameters:
         type: object
         additionalProperties: true
@@ -4045,6 +4104,10 @@ definitions:
           - ["data"]
           - ["data", "streams"]
           - ["data", "{{ parameters.name }}"]
+      default_values:
+        title: Default Values
+        description: placeholder
+        type: array
       $parameters:
         type: object
         additionalProperties: true
@@ -4056,7 +4119,11 @@ definitions:
         type: string
         enum: [ConfigComponentsResolver]
       stream_config:
-        "$ref": "#/definitions/StreamConfig"
+        anyOf:
+          - type: array
+            items:
+              "$ref": "#/definitions/StreamConfig"
+          - "$ref": "#/definitions/StreamConfig"
       components_mapping:
         type: array
         items:

airbyte_cdk/sources/declarative/extractors/__init__.py

@@ -2,8 +2,10 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+from airbyte_cdk.sources.declarative.extractors.combined_extractor import CombinedExtractor
 from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
 from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
+from airbyte_cdk.sources.declarative.extractors.key_value_extractor import KeyValueExtractor
 from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
 from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
 from airbyte_cdk.sources.declarative.extractors.response_to_file_extractor import (
@@ -18,4 +20,6 @@ __all__ = [
     "RecordFilter",
     "RecordSelector",
     "ResponseToFileExtractor",
+    "KeyValueExtractor",
+    "CombinedExtractor",
 ]

airbyte_cdk/sources/declarative/extractors/combined_extractor.py

@@ -0,0 +1,44 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from itertools import islice
+from typing import Any, Iterable, List, Mapping, MutableMapping, Union
+
+import dpath
+import requests
+
+from airbyte_cdk.sources.declarative.decoders import Decoder, JsonDecoder
+from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.types import Config
+
+
+@dataclass
+class CombinedExtractor(RecordExtractor):
+    """
+    Extractor that combines keys and values from two separate extractors.
+
+    The `keys_extractor` and `values_extractor` extract records independently
+    from the response. Their outputs are zipped together to form key-value mappings.
+
+    Each key from `keys_extractor` should correspond to a key in the resulting dictionary,
+    and each value from `values_extractor` is the value for that key.
+
+    Example:
+      keys_extractor -> yields: ["name", "age"]
+      values_extractor -> yields: ["Alice", 30]
+      result: { "name": "Alice", "age": 30 }
+    """
+
+    extractors: List[RecordExtractor]
+
+    def extract_records(self, response: requests.Response) -> Iterable[MutableMapping[Any, Any]]:
+        extractors_records = [extractor.extract_records(response) for extractor in self.extractors]
+
+        for records in zip(*extractors_records):
+            merged = {}
+            for record in records:
+                merged.update(record)  # merge all fields
+            yield merged

airbyte_cdk/sources/declarative/extractors/key_value_extractor.py

@@ -0,0 +1,46 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from itertools import islice
+from typing import Any, Iterable, List, Mapping, MutableMapping, Union
+
+import dpath
+import requests
+
+from airbyte_cdk.sources.declarative.decoders import Decoder, JsonDecoder
+from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.types import Config
+
+
+@dataclass
+class KeyValueExtractor(RecordExtractor):
+    """
+    Extractor that combines keys and values from two separate extractors.
+
+    The `keys_extractor` and `values_extractor` extract records independently
+    from the response. Their outputs are zipped together to form key-value mappings.
+
+    Each key from `keys_extractor` should correspond to a key in the resulting dictionary,
+    and each value from `values_extractor` is the value for that key.
+
+    Example:
+      keys_extractor -> yields: ["name", "age"]
+      values_extractor -> yields: ["Alice", 30]
+      result: { "name": "Alice", "age": 30 }
+    """
+
+    keys_extractor: RecordExtractor
+    values_extractor: RecordExtractor
+
+    def extract_records(self, response: requests.Response) -> Iterable[MutableMapping[Any, Any]]:
+        keys = list(self.keys_extractor.extract_records(response))
+        values = self.values_extractor.extract_records(response)
+
+        while True:
+            chunk = list(islice(values, len(keys)))
+            if not chunk:
+                break
+            yield dict(zip(keys, chunk))

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -262,9 +262,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
             }
         )
 
-        stream_configs = self._stream_configs(self._source_config) + self._dynamic_stream_configs(
-            self._source_config, config
-        )
+        stream_configs = self._stream_configs(self._source_config) + self.dynamic_streams
 
         api_budget_model = self._source_config.get("api_budget")
         if api_budget_model:

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1,5 +1,3 @@
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-
 # generated by datamodel-codegen:
 #   filename:  declarative_component_schema.yaml
 
@@ -1480,6 +1478,7 @@ class ComponentMappingDefinition(BaseModel):
         description="The expected data type of the value. If omitted, the type will be inferred from the value provided.",
         title="Value Type",
     )
+    create_or_update: Optional[bool] = False
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
@@ -1491,12 +1490,13 @@ class StreamConfig(BaseModel):
         examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]],
         title="Configs Pointer",
     )
+    default_values: Optional[List] = Field(None, description="placeholder", title="Default Values")
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
 class ConfigComponentsResolver(BaseModel):
     type: Literal["ConfigComponentsResolver"]
-    stream_config: StreamConfig
+    stream_config: Union[List[StreamConfig], StreamConfig]
     components_mapping: List[ComponentMappingDefinition]
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
@@ -1844,6 +1844,25 @@ class DefaultPaginator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class KeyValueExtractor(BaseModel):
+    type: Literal["KeyValueExtractor"]
+    keys_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field(
+        ..., description="placeholder"
+    )
+    values_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field(
+        ..., description="placeholder"
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
+class CombinedExtractor(BaseModel):
+    type: Literal["CombinedExtractor"]
+    extractors: List[
+        Union[DpathExtractor, CombinedExtractor, KeyValueExtractor, CustomRecordExtractor]
+    ] = Field(..., description="placeholder")
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class SessionTokenRequestApiKeyAuthenticator(BaseModel):
     type: Literal["ApiKey"]
     inject_into: RequestOption = Field(
@@ -1881,7 +1900,7 @@ class ListPartitionRouter(BaseModel):
 
 class RecordSelector(BaseModel):
     type: Literal["RecordSelector"]
-    extractor: Union[DpathExtractor, CustomRecordExtractor]
+    extractor: Union[DpathExtractor, CombinedExtractor, KeyValueExtractor, CustomRecordExtractor]
     record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field(
         None,
         description="Responsible for filtering records to be emitted by the Source.",
@@ -2164,7 +2183,7 @@ class DeclarativeStream(BaseModel):
         ]
     ] = Field(
         None,
-        description="Component used to retrieve the schema for the current stream.",
+        description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.",
         title="Schema Loader",
     )
     transformations: Optional[
@@ -2414,6 +2433,9 @@ class DynamicSchemaLoader(BaseModel):
         description="Component used to coordinate how records are extracted across stream slices and request pages.",
         title="Retriever",
     )
+    schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field(
+        None, description="placeholder", title="Schema Filter"
+    )
     schema_transformations: Optional[
         List[
             Union[
@@ -2749,6 +2771,7 @@ class DynamicDeclarativeStream(BaseModel):
 
 
 ComplexFieldType.update_forward_refs()
+CombinedExtractor.update_forward_refs()
 GzipDecoder.update_forward_refs()
 CompositeErrorHandler.update_forward_refs()
 DeclarativeSource1.update_forward_refs()

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -83,7 +83,9 @@ from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import (
     Parser,
 )
 from airbyte_cdk.sources.declarative.extractors import (
+    CombinedExtractor,
     DpathExtractor,
+    KeyValueExtractor,
     RecordFilter,
     RecordSelector,
     ResponseToFileExtractor,
@@ -142,6 +144,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     CheckStream as CheckStreamModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    CombinedExtractor as CombinedExtractorModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ComplexFieldType as ComplexFieldTypeModel,
 )
@@ -304,6 +309,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     KeysToSnakeCase as KeysToSnakeCaseModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    KeyValueExtractor as KeyValueExtractorModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     LegacySessionTokenAuthenticator as LegacySessionTokenAuthenticatorModel,
 )
@@ -641,6 +649,8 @@ class ModelToComponentFactory:
             DefaultErrorHandlerModel: self.create_default_error_handler,
             DefaultPaginatorModel: self.create_default_paginator,
             DpathExtractorModel: self.create_dpath_extractor,
+            KeyValueExtractorModel: self.create_key_value_extractor,
+            CombinedExtractorModel: self.create_combined_extractor,
             ResponseToFileExtractorModel: self.create_response_to_file_extractor,
             ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy,
             SessionTokenAuthenticatorModel: self.create_session_token_authenticator,
@@ -2218,6 +2228,36 @@ class ModelToComponentFactory:
             parameters=model.parameters or {},
         )
 
+    def create_key_value_extractor(
+        self,
+        model: KeyValueExtractorModel,
+        config: Config,
+        decoder: Optional[Decoder] = JsonDecoder(parameters={}),
+        **kwargs: Any,
+    ) -> KeyValueExtractor:
+        keys_extractor = self._create_component_from_model(
+            model=model.keys_extractor, decoder=decoder, config=config
+        )
+        values_extractor = self._create_component_from_model(
+            model=model.values_extractor, decoder=decoder, config=config
+        )
+
+        return KeyValueExtractor(keys_extractor=keys_extractor, values_extractor=values_extractor)
+
+    def create_combined_extractor(
+        self,
+        model: CombinedExtractorModel,
+        config: Config,
+        decoder: Optional[Decoder] = JsonDecoder(parameters={}),
+        **kwargs: Any,
+    ) -> CombinedExtractor:
+        extractors = [
+            self._create_component_from_model(model=extractor, decoder=decoder, config=config)
+            for extractor in model.extractors
+        ]
+
+        return CombinedExtractor(extractors=extractors)
+
     @staticmethod
     def create_response_to_file_extractor(
         model: ResponseToFileExtractorModel,
@@ -2432,10 +2472,14 @@ class ModelToComponentFactory:
         schema_type_identifier = self._create_component_from_model(
             model.schema_type_identifier, config=config, parameters=model.parameters or {}
         )
+        schema_filter = self._create_component_from_model(
+            model.schema_filter, config=config, parameters=model.parameters or {}
+        )
         return DynamicSchemaLoader(
             retriever=retriever,
             config=config,
             schema_transformations=schema_transformations,
+            schema_filter=schema_filter,
             schema_type_identifier=schema_type_identifier,
             parameters=model.parameters or {},
         )
@@ -3597,6 +3641,7 @@ class ModelToComponentFactory:
             field_path=field_path,  # type: ignore[arg-type] # field_path can be str and InterpolatedString
             value=interpolated_value,
             value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type),
+            create_or_update=model.create_or_update,
             parameters=model.parameters or {},
         )
 
@@ -3643,16 +3688,24 @@ class ModelToComponentFactory:
 
         return StreamConfig(
             configs_pointer=model_configs_pointer,
+            default_values=model.default_values,
             parameters=model.parameters or {},
         )
 
     def create_config_components_resolver(
         self, model: ConfigComponentsResolverModel, config: Config
     ) -> Any:
-        stream_config = self._create_component_from_model(
-            model.stream_config, config=config, parameters=model.parameters or {}
+        model_stream_configs = (
+            model.stream_config if isinstance(model.stream_config, list) else [model.stream_config]
         )
 
+        stream_configs = [
+            self._create_component_from_model(
+                stream_config, config=config, parameters=model.parameters or {}
+            )
+            for stream_config in model_stream_configs
+        ]
+
         components_mapping = [
             self._create_component_from_model(
                 model=components_mapping_definition_model,
@@ -3665,7 +3718,7 @@ class ModelToComponentFactory:
         ]
 
         return ConfigComponentsResolver(
-            stream_config=stream_config,
+            stream_configs=stream_configs,
             config=config,
             components_mapping=components_mapping,
             parameters=model.parameters or {},

airbyte_cdk/sources/declarative/resolvers/components_resolver.py

@@ -22,6 +22,7 @@ class ComponentMappingDefinition:
     value: Union["InterpolatedString", str]
     value_type: Optional[Type[Any]]
     parameters: InitVar[Mapping[str, Any]]
+    create_or_update: Optional[bool] = False
 
 
 @dataclass(frozen=True)
@@ -34,6 +35,7 @@ class ResolvedComponentMappingDefinition:
     value: "InterpolatedString"
     value_type: Optional[Type[Any]]
     parameters: InitVar[Mapping[str, Any]]
+    create_or_update: Optional[bool] = False
 
 
 @deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)

airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py

@@ -4,7 +4,8 @@
 
 from copy import deepcopy
 from dataclasses import InitVar, dataclass, field
-from typing import Any, Dict, Iterable, List, Mapping, Union
+from itertools import product
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Union
 
 import dpath
 from typing_extensions import deprecated
@@ -28,6 +29,7 @@ class StreamConfig:
 
     configs_pointer: List[Union[InterpolatedString, str]]
     parameters: InitVar[Mapping[str, Any]]
+    default_values: Optional[List[Any]] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self.configs_pointer = [
@@ -48,7 +50,7 @@ class ConfigComponentsResolver(ComponentsResolver):
         parameters (InitVar[Mapping[str, Any]]): Additional parameters for interpolation.
     """
 
-    stream_config: StreamConfig
+    stream_configs: List[StreamConfig]
     config: Config
     components_mapping: List[ComponentMappingDefinition]
     parameters: InitVar[Mapping[str, Any]]
@@ -82,6 +84,7 @@ class ConfigComponentsResolver(ComponentsResolver):
                         field_path=field_path,
                         value=interpolated_value,
                         value_type=component_mapping.value_type,
+                        create_or_update=component_mapping.create_or_update,
                         parameters=parameters,
                     )
                 )
@@ -91,17 +94,35 @@ class ConfigComponentsResolver(ComponentsResolver):
                 )
 
     @property
-    def _stream_config(self) -> Iterable[Mapping[str, Any]]:
-        path = [
-            node.eval(self.config) if not isinstance(node, str) else node
-            for node in self.stream_config.configs_pointer
-        ]
-        stream_config = dpath.get(dict(self.config), path, default=[])
-
-        if not isinstance(stream_config, list):
-            stream_config = [stream_config]
-
-        return stream_config
+    def _stream_config(self):
+        def resolve_path(pointer):
+            return [
+                node.eval(self.config) if not isinstance(node, str) else node for node in pointer
+            ]
+
+        def normalize_configs(configs):
+            return configs if isinstance(configs, list) else [configs]
+
+        def prepare_streams():
+            for stream_config in self.stream_configs:
+                path = resolve_path(stream_config.configs_pointer)
+                stream_configs = dpath.get(dict(self.config), path, default=[])
+                stream_configs = normalize_configs(stream_configs)
+                if stream_config.default_values:
+                    stream_configs += stream_config.default_values
+                yield [(i, item) for i, item in enumerate(stream_configs)]
+
+        def merge_combination(combo):
+            result = {}
+            for config_index, (elem_index, elem) in enumerate(combo):
+                if isinstance(elem, dict):
+                    result.update(elem)
+                else:
+                    result.setdefault(f"source_config_{config_index}", (elem_index, elem))
+            return result
+
+        all_indexed_streams = list(prepare_streams())
+        return [merge_combination(combo) for combo in product(*all_indexed_streams)]
 
     def resolve_components(
         self, stream_template_config: Dict[str, Any]
@@ -130,7 +151,21 @@ class ConfigComponentsResolver(ComponentsResolver):
                 )
 
                 path = [path.eval(self.config, **kwargs) for path in resolved_component.field_path]
+                parsed_value = self._parse_yaml_if_possible(value)
+                updated = dpath.set(updated_config, path, parsed_value)
 
-                dpath.set(updated_config, path, value)
+                if parsed_value and not updated and resolved_component.create_or_update:
+                    dpath.new(updated_config, path, parsed_value)
 
             yield updated_config
+
+    @staticmethod
+    def _parse_yaml_if_possible(value: Any) -> Any:
+        if isinstance(value, str):
+            try:
+                import yaml
+
+                return yaml.safe_load(value)
+            except Exception:
+                return value
+        return value

airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py

@@ -10,6 +10,7 @@ from typing import Any, List, Mapping, MutableMapping, Optional, Union
 import dpath
 from typing_extensions import deprecated
 
+from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
@@ -126,6 +127,7 @@ class DynamicSchemaLoader(SchemaLoader):
     parameters: InitVar[Mapping[str, Any]]
     schema_type_identifier: SchemaTypeIdentifier
     schema_transformations: List[RecordTransformation] = field(default_factory=lambda: [])
+    schema_filter: Optional[RecordFilter] = None
 
     def get_json_schema(self) -> Mapping[str, Any]:
         """
@@ -151,20 +153,18 @@ class DynamicSchemaLoader(SchemaLoader):
             )
             properties[key] = value
 
-        transformed_properties = self._transform(properties, {})
+        filtred_transformed_properties = self._transform(self._filter(properties))
 
         return {
             "$schema": "https://json-schema.org/draft-07/schema#",
             "type": "object",
             "additionalProperties": True,
-            "properties": transformed_properties,
+            "properties": filtred_transformed_properties,
         }
 
     def _transform(
         self,
         properties: Mapping[str, Any],
-        stream_state: StreamState,
-        stream_slice: Optional[StreamSlice] = None,
     ) -> Mapping[str, Any]:
         for transformation in self.schema_transformations:
             transformation.transform(
@@ -173,6 +173,21 @@ class DynamicSchemaLoader(SchemaLoader):
             )
         return properties
 
+    def _filter(
+        self,
+        properties: Mapping[str, Any],
+    ) -> Mapping[str, Any]:
+        if not self.schema_filter:
+            return properties
+
+        filtered_properties = {}
+        for item in self.schema_filter.filter_records(
+            ({k: v} for k, v in properties.items()),
+            {},
+        ):
+            filtered_properties.update(item)
+        return filtered_properties
+
     def _get_key(
         self,
         raw_schema: MutableMapping[str, Any],

{airbyte_cdk-6.48.15.dist-info → airbyte_cdk-6.48.17.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.48.15
+Version: 6.48.17.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.48.15.dist-info → airbyte_cdk-6.48.17.dev0.dist-info}/RECORD

@@ -1,10 +1,10 @@
 airbyte_cdk/__init__.py,sha256=52uncJvDQNHvwKxaqzXgnMYTptIl65LDJr2fvlk8-DU,11707
 airbyte_cdk/cli/__init__.py,sha256=CXsai3MYMLZ_sqi2vPAIVcKDun8VRqlv0cKffBI0iSY,346
 airbyte_cdk/cli/airbyte_cdk/__init__.py,sha256=8IoEcbdYr7CMAh97Xut5__uHH9vV4LKUtSBNTk3qEWY,2031
-airbyte_cdk/cli/airbyte_cdk/_connector.py,sha256=drKb_EXJOFX-cSeLwJB8WXE-lesOL0dx2ziWSmW3Jkg,5187
+airbyte_cdk/cli/airbyte_cdk/_connector.py,sha256=PA1beSvhcvm-6watFCrjNat9GC4RpYolC53z2r1V3dc,5234
 airbyte_cdk/cli/airbyte_cdk/_image.py,sha256=AkBEZrRYXEwvhW7hPOPRWeYEZutzi2PIzjpl7_yaE8I,2890
 airbyte_cdk/cli/airbyte_cdk/_manifest.py,sha256=aFdeeWgek7oXR3YfZPxk7kBZ64Blmsr0dAXN6BVGiIA,482
-airbyte_cdk/cli/airbyte_cdk/_secrets.py,sha256=1IqVz-csoMJoKajSX4DzoWrngswuNHBPkzzchQggAeE,17010
+airbyte_cdk/cli/airbyte_cdk/_secrets.py,sha256=jLtpkhFJHavABN3UuqddeDRGS8v1iEj0e_f6WTeYoQA,17409
 airbyte_cdk/cli/airbyte_cdk/_version.py,sha256=ohZNIktLFk91sdzqFW5idaNrZAPX2dIRnz---_fcKOE,352
 airbyte_cdk/cli/airbyte_cdk/exceptions.py,sha256=bsGmlWN6cXL2jCD1WYAZMqFmK1OLg2xLrcC_60KHSeA,803
 airbyte_cdk/cli/source_declarative_manifest/__init__.py,sha256=-0ST722Nj65bgRokzpzPkD1NBBW5CytEHFUe38cB86Q,91
@@ -89,7 +89,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=GoZJ8Oxb
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=7tB0-d4lbI97GB5lBI6-qezxLoKZN4QeN_4Vht9TDFo,167270
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=o3t_npbhuPBKJ4PCTYz0zh3XT6AtpDMhe8TMujPYCUs,169241
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=qmyMnnet92eGc3C22yBtpvD5UZjqdhsAafP_zxI5wp8,1814
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
@@ -102,9 +102,11 @@ airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py,sha256=
 airbyte_cdk/sources/declarative/decoders/xml_decoder.py,sha256=EU-7t-5vIGRHZ14h-f0GUE4V5-eTM9Flux-A8xgI1Rc,3117
 airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py,sha256=Din18m61aly2oG6TaXGpLcbfUHVOzjGzuMYkyxfHXT4,2290
 airbyte_cdk/sources/declarative/exceptions.py,sha256=kTPUA4I2NV4J6HDz-mKPGMrfuc592akJnOyYx38l_QM,176
-airbyte_cdk/sources/declarative/extractors/__init__.py,sha256=RmV-IkO1YLj0PSOrrqC9AV1gO8-90t8UTDVfJGshN9E,754
+airbyte_cdk/sources/declarative/extractors/__init__.py,sha256=3Cu6FduH2FmWigvMAMd03o_ZfOSoUFLSb3BBfoc1xW4,989
+airbyte_cdk/sources/declarative/extractors/combined_extractor.py,sha256=_yM2eywOYYXDkONPssGU6JkEJlc81-cfamiEw7t_suk,1603
 airbyte_cdk/sources/declarative/extractors/dpath_extractor.py,sha256=wR4Ol4MG2lt5UlqXF5EU_k7qa5cN4_-luu3PJ1PlO3A,3131
 airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8VwkjS0xD_u350Km3SiYP7hpOOgiLg5o,1169
+airbyte_cdk/sources/declarative/extractors/key_value_extractor.py,sha256=wmAr5RqOQXHsqhHDOwAIhPZZSRYm_NnPn3xTxl3oHyM,1641
 airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
 airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
 airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=vCpwX1PVRFPYKMzm0DHHP3YEZ0Gmd3bBzggOsRha038,7192
@@ -127,20 +129,20 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkH
 airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
 airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
 airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=UYSJ5gW7TkHALYnNvUnRP3RlyGwGuRMObF3BHuNzjJM,5320
-airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=cZNUOeIogrCmCS7RXeJqQIlnsANigz1cngpLko02M2g,23191
+airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=88uuVKamdAFzsC3JBKvajI8icZj00BmRLFQe9bFskrk,23132
 airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=V2lpYE9LJKvz6BUViHk4vaRGndxNABmPbDCtyYdkqaE,4013
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
 airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py,sha256=Imnj3yef0aqRdLfaUxkIYISUb8YkiPrRH_wBd-x8HjM,5999
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=FjZ2WrItKCqMVxc14XRbMPu11Jhl1Ds3Lugy_2wPoQ8,118235
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=8La2uX0ODkXwBW1PXkHpg-EkSwttf_nDSXYq9BiPKVo,119463
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9Rbu5OexXSDN9QWDo8YU4tT9M2LDVOgGA,802
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=2UdpCz3yi7ISZTyqkQXSSy3dMxeyOWqV7OlAS5b9GVg,11568
 airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=uPefJHWYdLxupSVqI4_VnU30JlbA44EQ7GwY0RdfaVw,168443
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=ZUEh2-mIuhLBjZrECyZB54K-S402sjHDXIUcJ1Cwy18,170489
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -194,8 +196,8 @@ airbyte_cdk/sources/declarative/requesters/request_options/request_options_provi
 airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
 airbyte_cdk/sources/declarative/requesters/requester.py,sha256=T6tMx_Bx4iT-0YVjY7IzgRil-gaIu9n01b1iwpTh3Ek,5516
 airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=NiDcz5qi8HPsfX94MUmnX0Rgs_kQXGvucOmJjNWlxKQ,1207
-airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb9artL4ZkeqN8RmEykHH6FJgqXD7fCEnh1X0,1936
-airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
+airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=oJIpy66ep8n-QOc8GwpllApTRcl4QtQhkTw5fWWra2w,2026
+airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=1coEpB_mRAV1NgoaWd4CMllf4SPOFQGi3ac3DTP71ZQ,6791
 airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=nQepwG_RfW53sgwvK5dLPqfCx0VjsQ83nYoPjBMAaLM,527
 airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=6oZtnCHm9NdDvjTSrVwPQOXGSdETSIR7eWH2vFjM7jI,4855
@@ -211,7 +213,7 @@ airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=Ui7V_etQ-s
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
 airbyte_cdk/sources/declarative/schema/composite_schema_loader.py,sha256=ymGbvxS_QyGc4nnjEyRo5ch8bVedELO41PAUxKXZyMw,1113
 airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=UnbzlExmwoQiVV8zDg4lhAEaqA_0pRfwbMRe8yqOuWk,1834
-airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=J8Q_iJYhcSQLWyt0bTZCbDAGpxt9G8FCc6Q9jtGsNzw,10703
+airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=q_AUnbzm6AoZd31NhlU4ucTkwRwEqE5VvVeXqKLYfrM,11186
 airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRsatRJq3R3BeyRR0wIoK3gcP1gcpVRQ_P5U,464
 airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py,sha256=5Wl-fqW-pVf_dxJ4yGHMAFfC4JjKHYJhqFJT1xA57F4,4177
 airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLnrDLxf1PJKdUqvQq2RVnAOAzNSY,379
@@ -419,9 +421,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.48.15.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.48.15.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.48.15.dist-info/METADATA,sha256=DP9YSnAFjf7aaIFb4V1xGPe6kc3TY8HAQNZdy7Bgaq4,6344
-airbyte_cdk-6.48.15.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.48.15.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.48.15.dist-info/RECORD,,
+airbyte_cdk-6.48.17.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.48.17.dev0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.48.17.dev0.dist-info/METADATA,sha256=rybxpJSXyko7F2tXrJGvD79wVY9N0ie5ibd0CXaNE8w,6349
+airbyte_cdk-6.48.17.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.48.17.dev0.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.48.17.dev0.dist-info/RECORD,,