airbyte-cdk 6.43.0.dev0__tar.gz → 6.43.1__tar.gz

{airbyte_cdk-6.43.0.dev0 → airbyte_cdk-6.43.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.43.0.dev0
+Version: 6.43.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.43.0.dev0 → airbyte_cdk-6.43.1}/airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -4,7 +4,7 @@
 
 
 from dataclasses import asdict, dataclass, field
-from typing import Any, List, Mapping
+from typing import Any, Dict, List, Mapping
 
 from airbyte_cdk.connector_builder.test_reader import TestReader
 from airbyte_cdk.models import (
@@ -27,30 +27,34 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE = 5
 DEFAULT_MAXIMUM_NUMBER_OF_SLICES = 5
 DEFAULT_MAXIMUM_RECORDS = 100
+DEFAULT_MAXIMUM_STREAMS = 100
 
 MAX_PAGES_PER_SLICE_KEY = "max_pages_per_slice"
 MAX_SLICES_KEY = "max_slices"
 MAX_RECORDS_KEY = "max_records"
+MAX_STREAMS_KEY = "max_streams"
 
 
 @dataclass
-class TestReadLimits:
+class TestLimits:
     max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS)
     max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE)
     max_slices: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_SLICES)
+    max_streams: int = field(default=DEFAULT_MAXIMUM_STREAMS)
 
 
-def get_limits(config: Mapping[str, Any]) -> TestReadLimits:
+def get_limits(config: Mapping[str, Any]) -> TestLimits:
     command_config = config.get("__test_read_config", {})
     max_pages_per_slice = (
         command_config.get(MAX_PAGES_PER_SLICE_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE
     )
     max_slices = command_config.get(MAX_SLICES_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_SLICES
     max_records = command_config.get(MAX_RECORDS_KEY) or DEFAULT_MAXIMUM_RECORDS
-    return TestReadLimits(max_records, max_pages_per_slice, max_slices)
+    max_streams = command_config.get(MAX_STREAMS_KEY) or DEFAULT_MAXIMUM_STREAMS
+    return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams)
 
 
-def create_source(config: Mapping[str, Any], limits: TestReadLimits) -> ManifestDeclarativeSource:
+def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDeclarativeSource:
     manifest = config["__injected_declarative_manifest"]
     return ManifestDeclarativeSource(
         config=config,
@@ -71,7 +75,7 @@ def read_stream(
     config: Mapping[str, Any],
     configured_catalog: ConfiguredAirbyteCatalog,
     state: List[AirbyteStateMessage],
-    limits: TestReadLimits,
+    limits: TestLimits,
 ) -> AirbyteMessage:
     try:
         test_read_handler = TestReader(
@@ -117,13 +121,23 @@ def resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
         return error.as_airbyte_message()
 
 
-def full_resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
+def full_resolve_manifest(source: ManifestDeclarativeSource, limits: TestLimits) -> AirbyteMessage:
     try:
         manifest = {**source.resolved_manifest}
         streams = manifest.get("streams", [])
         for stream in streams:
             stream["dynamic_stream_name"] = None
-        streams.extend(source.dynamic_streams)
+
+        mapped_streams: Dict[str, List[Dict[str, Any]]] = {}
+        for stream in source.dynamic_streams:
+            generated_streams = mapped_streams.setdefault(stream["dynamic_stream_name"], [])
+
+            if len(generated_streams) < limits.max_streams:
+                generated_streams += [stream]
+
+        for generated_streams_list in mapped_streams.values():
+            streams.extend(generated_streams_list)
+
         manifest["streams"] = streams
         return AirbyteMessage(
             type=Type.RECORD,

{airbyte_cdk-6.43.0.dev0 → airbyte_cdk-6.43.1}/airbyte_cdk/connector_builder/main.py

@@ -10,7 +10,7 @@ import orjson
 
 from airbyte_cdk.connector import BaseConnector
 from airbyte_cdk.connector_builder.connector_builder_handler import (
-    TestReadLimits,
+    TestLimits,
     create_source,
     full_resolve_manifest,
     get_limits,
@@ -73,7 +73,7 @@ def handle_connector_builder_request(
     config: Mapping[str, Any],
     catalog: Optional[ConfiguredAirbyteCatalog],
     state: List[AirbyteStateMessage],
-    limits: TestReadLimits,
+    limits: TestLimits,
 ) -> AirbyteMessage:
     if command == "resolve_manifest":
         return resolve_manifest(source)
@@ -83,7 +83,7 @@ def handle_connector_builder_request(
         ), "`test_read` requires a valid `ConfiguredAirbyteCatalog`, got None."
         return read_stream(source, config, catalog, state, limits)
     elif command == "full_resolve_manifest":
-        return full_resolve_manifest(source)
+        return full_resolve_manifest(source, limits)
     else:
         raise ValueError(f"Unrecognized command {command}.")
 

{airbyte_cdk-6.43.0.dev0 → airbyte_cdk-6.43.1}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1023,15 +1023,6 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
-  EmitPartialRecordMergeStrategy:
-    title: Emit Partial Record
-    description: Record merge strategy where in the case where multiple requests are needed to retrieve all properties, properties are not consolidated back into a single record and are instead emitted as separate groups of properties. This strategy should only be used when records do not have a unique identifier like a primary key.
-    required:
-      - type
-    properties:
-      type:
-        type: string
-        enum: [EmitPartialRecordMergeStrategy]
   JwtAuthenticator:
     title: JWT Authenticator
     description: Authenticator for requests using JWT authentication flow.
@@ -1740,30 +1731,6 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
-  GroupByKeyMergeStrategy:
-    title: Group by Key
-    description: Record merge strategy that combines records according to fields on the record.
-    required:
-      - type
-      - key
-    properties:
-      type:
-        type: string
-        enum: [GroupByKeyMergeStrategy]
-      key:
-        title: Key
-        description: The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.
-        anyOf:
-          - type: string
-          - type: array
-            items:
-              type: string
-        examples:
-          - "id"
-          - ["parent_id", "end_date"]
-      $parameters:
-        type: object
-        additionalProperties: true
   SessionTokenAuthenticator:
     type: object
     required:
@@ -1983,9 +1950,7 @@ definitions:
           - type: string
           - type: object
             additionalProperties:
-              anyOf:
-                - type: string
-                - $ref": "#/definitions/QueryProperties"
+              type: string
         interpolation_context:
           - next_page_token
           - stream_interval
@@ -3003,98 +2968,6 @@ definitions:
     examples:
       - id
       - ["code", "type"]
-  PropertiesFromEndpoint:
-    title: Properties from Endpoint
-    description: Defines the behavior for fetching the list of properties from an API that will be loaded into the requests to extract records.
-    type: object
-    required:
-      - type
-      - property_field_path
-      - retriever
-    properties:
-      type:
-        type: string
-        enum: [PropertiesFromEndpoint]
-      property_field_path:
-        description: Describes the path to the field that should be extracted
-        type: array
-        items:
-          type: string
-        examples:
-          - ["name"]
-        interpolation_context:
-          - config
-          - parameters
-      retriever:
-        description: Requester component that describes how to fetch the properties to query from a remote API endpoint.
-        anyOf:
-          - "$ref": "#/definitions/CustomRetriever"
-          - "$ref": "#/definitions/SimpleRetriever"
-      $parameters:
-        type: object
-        additionalProperties: true
-  PropertyChunking:
-    title: Property Chunking
-    description: For APIs with restrictions on the amount of properties that can be requester per request, property chunking can be applied to make multiple requests with a subset of the properties.
-    type: object
-    required:
-      - type
-      - property_limit_type
-    properties:
-      type:
-        type: string
-        enum: [PropertyChunking]
-      property_limit_type:
-        title: Property Limit Type
-        description: The type used to determine the maximum number of properties per chunk
-        enum:
-          - characters
-          - property_count
-      property_limit:
-        title: Property Limit
-        description: The maximum amount of properties that can be retrieved per request according to the limit type.
-        type: integer
-      record_merge_strategy:
-        title: Record Merge Strategy
-        description: Dictates how to records that require multiple requests to get all properties should be emitted to the destination
-        anyOf:
-          - "$ref": "#/definitions/EmitPartialRecordMergeStrategy"
-          - "$ref": "#/definitions/GroupByKeyMergeStrategy"
-      $parameters:
-        type: object
-        additionalProperties: true
-  QueryProperties:
-    title: Query Properties
-    description: For APIs that require explicit specification of the properties to query for, this component specifies which property fields and how they are supplied to outbound requests.
-    type: object
-    required:
-      - type
-      - property_list
-    properties:
-      type:
-        type: string
-        enum: [QueryProperties]
-      property_list:
-        title: Property List
-        description: The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint
-        anyOf:
-          - type: array
-            items:
-              type: string
-          - "$ref": "#/definitions/PropertiesFromEndpoint"
-      always_include_properties:
-        title: Always Include Properties
-        description: The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.
-        type: array
-        items:
-          type: string
-      property_chunking:
-        title: Property Chunking
-        description: Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.
-        "$ref": "#/definitions/PropertyChunking"
-      $parameters:
-        type: object
-        additionalProperties: true
   RecordFilter:
     title: Record Filter
     description: Filter applied on a list of records.

{airbyte_cdk-6.43.0.dev0 → airbyte_cdk-6.43.1}/airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -397,6 +397,12 @@ class ManifestDeclarativeSource(DeclarativeSource):
             for dynamic_stream in components_resolver.resolve_components(
                 stream_template_config=stream_template_config
             ):
+                dynamic_stream = {
+                    **ManifestComponentTransformer().propagate_types_and_parameters(
+                        "", dynamic_stream, {}, use_parent_parameters=True
+                    )
+                }
+
                 if "type" not in dynamic_stream:
                     dynamic_stream["type"] = "DeclarativeStream"
 

{airbyte_cdk-6.43.0.dev0 → airbyte_cdk-6.43.1}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1,5 +1,3 @@
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-
 # generated by datamodel-codegen:
 #   filename:  declarative_component_schema.yaml
 
@@ -345,10 +343,6 @@ class Clamping(BaseModel):
     target_details: Optional[Dict[str, Any]] = None
 
 
-class EmitPartialRecordMergeStrategy(BaseModel):
-    type: Literal["EmitPartialRecordMergeStrategy"]
-
-
 class Algorithm(Enum):
     HS256 = "HS256"
     HS384 = "HS384"
@@ -722,17 +716,6 @@ class ExponentialBackoffStrategy(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class GroupByKeyMergeStrategy(BaseModel):
-    type: Literal["GroupByKeyMergeStrategy"]
-    key: Union[str, List[str]] = Field(
-        ...,
-        description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.",
-        examples=["id", ["parent_id", "end_date"]],
-        title="Key",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class SessionTokenRequestBearerAuthenticator(BaseModel):
     type: Literal["Bearer"]
 
@@ -1204,33 +1187,6 @@ class PrimaryKey(BaseModel):
     )
 
 
-class PropertyLimitType(Enum):
-    characters = "characters"
-    property_count = "property_count"
-
-
-class PropertyChunking(BaseModel):
-    type: Literal["PropertyChunking"]
-    property_limit_type: PropertyLimitType = Field(
-        ...,
-        description="The type used to determine the maximum number of properties per chunk",
-        title="Property Limit Type",
-    )
-    property_limit: Optional[int] = Field(
-        None,
-        description="The maximum amount of properties that can be retrieved per request according to the limit type.",
-        title="Property Limit",
-    )
-    record_merge_strategy: Optional[
-        Union[EmitPartialRecordMergeStrategy, GroupByKeyMergeStrategy]
-    ] = Field(
-        None,
-        description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination",
-        title="Record Merge Strategy",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class RecordFilter(BaseModel):
     type: Literal["RecordFilter"]
     condition: Optional[str] = Field(
@@ -2218,7 +2174,7 @@ class HttpRequester(BaseModel):
         examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
         title="Request Headers",
     )
-    request_parameters: Optional[Union[str, Dict[str, Union[str, Any]]]] = Field(
+    request_parameters: Optional[Union[str, Dict[str, str]]] = Field(
         None,
         description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
         examples=[
@@ -2308,40 +2264,6 @@ class ParentStreamConfig(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class PropertiesFromEndpoint(BaseModel):
-    type: Literal["PropertiesFromEndpoint"]
-    property_field_path: List[str] = Field(
-        ...,
-        description="Describes the path to the field that should be extracted",
-        examples=[["name"]],
-    )
-    retriever: Union[CustomRetriever, SimpleRetriever] = Field(
-        ...,
-        description="Requester component that describes how to fetch the properties to query from a remote API endpoint.",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
-class QueryProperties(BaseModel):
-    type: Literal["QueryProperties"]
-    property_list: Union[List[str], PropertiesFromEndpoint] = Field(
-        ...,
-        description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint",
-        title="Property List",
-    )
-    always_include_properties: Optional[List[str]] = Field(
-        None,
-        description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.",
-        title="Always Include Properties",
-    )
-    property_chunking: Optional[PropertyChunking] = Field(
-        None,
-        description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.",
-        title="Property Chunking",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class StateDelegatingStream(BaseModel):
     type: Literal["StateDelegatingStream"]
     name: str = Field(..., description="The stream name.", example=["Users"], title="Name")
@@ -2590,6 +2512,5 @@ DeclarativeStream.update_forward_refs()
 SessionTokenAuthenticator.update_forward_refs()
 DynamicSchemaLoader.update_forward_refs()
 ParentStreamConfig.update_forward_refs()
-PropertiesFromEndpoint.update_forward_refs()
 SimpleRetriever.update_forward_refs()
 AsyncRetriever.update_forward_refs()

{airbyte_cdk-6.43.0.dev0 → airbyte_cdk-6.43.1}/airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py

@@ -4,7 +4,7 @@
 
 import copy
 import typing
-from typing import Any, Mapping
+from typing import Any, Mapping, Optional
 
 PARAMETERS_STR = "$parameters"
 
@@ -94,6 +94,7 @@ class ManifestComponentTransformer:
         parent_field_identifier: str,
         declarative_component: Mapping[str, Any],
         parent_parameters: Mapping[str, Any],
+        use_parent_parameters: Optional[bool] = None,
     ) -> Mapping[str, Any]:
         """
         Recursively transforms the specified declarative component and subcomponents to propagate parameters and insert the
@@ -103,6 +104,7 @@ class ManifestComponentTransformer:
         :param declarative_component: The current component that is having type and parameters added
         :param parent_field_identifier: The name of the field of the current component coming from the parent component
         :param parent_parameters: The parameters set on parent components defined before the current component
+        :param use_parent_parameters: If set, parent parameters will be used as the source of truth when key names are the same
         :return: A deep copy of the transformed component with types and parameters persisted to it
         """
         propagated_component = dict(copy.deepcopy(declarative_component))
@@ -130,7 +132,11 @@ class ManifestComponentTransformer:
         # level take precedence
         current_parameters = dict(copy.deepcopy(parent_parameters))
         component_parameters = propagated_component.pop(PARAMETERS_STR, {})
-        current_parameters = {**current_parameters, **component_parameters}
+        current_parameters = (
+            {**component_parameters, **current_parameters}
+            if use_parent_parameters
+            else {**current_parameters, **component_parameters}
+        )
 
         # Parameters should be applied to the current component fields with the existing field taking precedence over parameters if
         # both exist
@@ -145,7 +151,10 @@ class ManifestComponentTransformer:
                 excluded_parameter = current_parameters.pop(field_name, None)
                 parent_type_field_identifier = f"{propagated_component.get('type')}.{field_name}"
                 propagated_component[field_name] = self.propagate_types_and_parameters(
-                    parent_type_field_identifier, field_value, current_parameters
+                    parent_type_field_identifier,
+                    field_value,
+                    current_parameters,
+                    use_parent_parameters=use_parent_parameters,
                 )
                 if excluded_parameter:
                     current_parameters[field_name] = excluded_parameter
@@ -158,7 +167,10 @@ class ManifestComponentTransformer:
                             f"{propagated_component.get('type')}.{field_name}"
                         )
                         field_value[i] = self.propagate_types_and_parameters(
-                            parent_type_field_identifier, element, current_parameters
+                            parent_type_field_identifier,
+                            element,
+                            current_parameters,
+                            use_parent_parameters=use_parent_parameters,
                         )
                 if excluded_parameter:
                     current_parameters[field_name] = excluded_parameter