airbyte-cdk 6.44.0__tar.gz → 6.45.0__tar.gz

{airbyte_cdk-6.44.0 → airbyte_cdk-6.45.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.44.0
+Version: 6.45.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.44.0 → airbyte_cdk-6.45.0}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -341,7 +341,7 @@ definitions:
     properties:
       type:
         type: string
-        enum: [ DynamicStreamCheckConfig ]
+        enum: [DynamicStreamCheckConfig]
       dynamic_stream_name:
         title: Dynamic Stream Name
         description: The dynamic stream name.
@@ -1752,6 +1752,30 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  GroupByKeyMergeStrategy:
+    title: Group by Key
+    description: Record merge strategy that combines records according to fields on the record.
+    required:
+      - type
+      - key
+    properties:
+      type:
+        type: string
+        enum: [GroupByKeyMergeStrategy]
+      key:
+        title: Key
+        description: The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.
+        anyOf:
+          - type: string
+          - type: array
+            items:
+              type: string
+        examples:
+          - "id"
+          - ["parent_id", "end_date"]
+      $parameters:
+        type: object
+        additionalProperties: true
   SessionTokenAuthenticator:
     type: object
     required:
@@ -1971,7 +1995,9 @@ definitions:
           - type: string
           - type: object
             additionalProperties:
-              type: string
+              anyOf:
+                - type: string
+                - $ref": "#/definitions/QueryProperties"
         interpolation_context:
           - next_page_token
           - stream_interval
@@ -2989,6 +3015,96 @@ definitions:
     examples:
       - id
       - ["code", "type"]
+  PropertiesFromEndpoint:
+    title: Properties from Endpoint
+    description: Defines the behavior for fetching the list of properties from an API that will be loaded into the requests to extract records.
+    type: object
+    required:
+      - type
+      - property_field_path
+      - retriever
+    properties:
+      type:
+        type: string
+        enum: [PropertiesFromEndpoint]
+      property_field_path:
+        description: Describes the path to the field that should be extracted
+        type: array
+        items:
+          type: string
+        examples:
+          - ["name"]
+        interpolation_context:
+          - config
+          - parameters
+      retriever:
+        description: Requester component that describes how to fetch the properties to query from a remote API endpoint.
+        anyOf:
+          - "$ref": "#/definitions/CustomRetriever"
+          - "$ref": "#/definitions/SimpleRetriever"
+      $parameters:
+        type: object
+        additionalProperties: true
+  PropertyChunking:
+    title: Property Chunking
+    description: For APIs with restrictions on the amount of properties that can be requester per request, property chunking can be applied to make multiple requests with a subset of the properties.
+    type: object
+    required:
+      - type
+      - property_limit_type
+    properties:
+      type:
+        type: string
+        enum: [PropertyChunking]
+      property_limit_type:
+        title: Property Limit Type
+        description: The type used to determine the maximum number of properties per chunk
+        enum:
+          - characters
+          - property_count
+      property_limit:
+        title: Property Limit
+        description: The maximum amount of properties that can be retrieved per request according to the limit type.
+        type: integer
+      record_merge_strategy:
+        title: Record Merge Strategy
+        description: Dictates how to records that require multiple requests to get all properties should be emitted to the destination
+        "$ref": "#/definitions/GroupByKeyMergeStrategy"
+      $parameters:
+        type: object
+        additionalProperties: true
+  QueryProperties:
+    title: Query Properties
+    description: For APIs that require explicit specification of the properties to query for, this component specifies which property fields and how they are supplied to outbound requests.
+    type: object
+    required:
+      - type
+      - property_list
+    properties:
+      type:
+        type: string
+        enum: [QueryProperties]
+      property_list:
+        title: Property List
+        description: The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint
+        anyOf:
+          - type: array
+            items:
+              type: string
+          - "$ref": "#/definitions/PropertiesFromEndpoint"
+      always_include_properties:
+        title: Always Include Properties
+        description: The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.
+        type: array
+        items:
+          type: string
+      property_chunking:
+        title: Property Chunking
+        description: Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.
+        "$ref": "#/definitions/PropertyChunking"
+      $parameters:
+        type: object
+        additionalProperties: true
   RecordFilter:
     title: Record Filter
     description: Filter applied on a list of records.

{airbyte_cdk-6.44.0 → airbyte_cdk-6.45.0}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+
 # generated by datamodel-codegen:
 #   filename:  declarative_component_schema.yaml
 
@@ -49,7 +51,7 @@ class DynamicStreamCheckConfig(BaseModel):
     )
     stream_count: Optional[int] = Field(
         0,
-        description="Numbers of the streams to try reading from when running a check operation.",
+        description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.",
         title="Stream Count",
     )
 
@@ -718,6 +720,17 @@ class ExponentialBackoffStrategy(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class GroupByKeyMergeStrategy(BaseModel):
+    type: Literal["GroupByKeyMergeStrategy"]
+    key: Union[str, List[str]] = Field(
+        ...,
+        description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.",
+        examples=["id", ["parent_id", "end_date"]],
+        title="Key",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class SessionTokenRequestBearerAuthenticator(BaseModel):
     type: Literal["Bearer"]
 
@@ -1189,6 +1202,31 @@ class PrimaryKey(BaseModel):
     )
 
 
+class PropertyLimitType(Enum):
+    characters = "characters"
+    property_count = "property_count"
+
+
+class PropertyChunking(BaseModel):
+    type: Literal["PropertyChunking"]
+    property_limit_type: PropertyLimitType = Field(
+        ...,
+        description="The type used to determine the maximum number of properties per chunk",
+        title="Property Limit Type",
+    )
+    property_limit: Optional[int] = Field(
+        None,
+        description="The maximum amount of properties that can be retrieved per request according to the limit type.",
+        title="Property Limit",
+    )
+    record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field(
+        None,
+        description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination",
+        title="Record Merge Strategy",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class RecordFilter(BaseModel):
     type: Literal["RecordFilter"]
     condition: Optional[str] = Field(
@@ -2187,7 +2225,7 @@ class HttpRequester(BaseModel):
         examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
         title="Request Headers",
     )
-    request_parameters: Optional[Union[str, Dict[str, str]]] = Field(
+    request_parameters: Optional[Union[str, Dict[str, Union[str, Any]]]] = Field(
         None,
         description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
         examples=[
@@ -2277,6 +2315,40 @@ class ParentStreamConfig(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class PropertiesFromEndpoint(BaseModel):
+    type: Literal["PropertiesFromEndpoint"]
+    property_field_path: List[str] = Field(
+        ...,
+        description="Describes the path to the field that should be extracted",
+        examples=[["name"]],
+    )
+    retriever: Union[CustomRetriever, SimpleRetriever] = Field(
+        ...,
+        description="Requester component that describes how to fetch the properties to query from a remote API endpoint.",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
+class QueryProperties(BaseModel):
+    type: Literal["QueryProperties"]
+    property_list: Union[List[str], PropertiesFromEndpoint] = Field(
+        ...,
+        description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint",
+        title="Property List",
+    )
+    always_include_properties: Optional[List[str]] = Field(
+        None,
+        description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.",
+        title="Always Include Properties",
+    )
+    property_chunking: Optional[PropertyChunking] = Field(
+        None,
+        description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.",
+        title="Property Chunking",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class StateDelegatingStream(BaseModel):
     type: Literal["StateDelegatingStream"]
     name: str = Field(..., description="The stream name.", example=["Users"], title="Name")
@@ -2525,5 +2597,6 @@ DeclarativeStream.update_forward_refs()
 SessionTokenAuthenticator.update_forward_refs()
 DynamicSchemaLoader.update_forward_refs()
 ParentStreamConfig.update_forward_refs()
+PropertiesFromEndpoint.update_forward_refs()
 SimpleRetriever.update_forward_refs()
 AsyncRetriever.update_forward_refs()

{airbyte_cdk-6.44.0 → airbyte_cdk-6.45.0}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
 
 from __future__ import annotations
@@ -234,6 +234,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     FlattenFields as FlattenFieldsModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    GroupByKeyMergeStrategy as GroupByKeyMergeStrategyModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     GroupingPartitionRouter as GroupingPartitionRouterModel,
 )
@@ -324,6 +327,18 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ParentStreamConfig as ParentStreamConfigModel,
 )
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    PropertiesFromEndpoint as PropertiesFromEndpointModel,
+)
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    PropertyChunking as PropertyChunkingModel,
+)
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    PropertyLimitType as PropertyLimitTypeModel,
+)
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    QueryProperties as QueryPropertiesModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     Rate as RateModel,
 )
@@ -432,6 +447,17 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies import (
     PageIncrement,
     StopConditionPaginationStrategyDecorator,
 )
+from airbyte_cdk.sources.declarative.requesters.query_properties import (
+    PropertiesFromEndpoint,
+    PropertyChunking,
+    QueryProperties,
+)
+from airbyte_cdk.sources.declarative.requesters.query_properties.property_chunking import (
+    PropertyLimitType,
+)
+from airbyte_cdk.sources.declarative.requesters.query_properties.strategies import (
+    GroupByKey,
+)
 from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType
 from airbyte_cdk.sources.declarative.requesters.request_options import (
     DatetimeBasedRequestOptionsProvider,
@@ -596,6 +622,7 @@ class ModelToComponentFactory:
             ResponseToFileExtractorModel: self.create_response_to_file_extractor,
             ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy,
             SessionTokenAuthenticatorModel: self.create_session_token_authenticator,
+            GroupByKeyMergeStrategyModel: self.create_group_by_key,
             HttpRequesterModel: self.create_http_requester,
             HttpResponseFilterModel: self.create_http_response_filter,
             InlineSchemaLoaderModel: self.create_inline_schema_loader,
@@ -625,6 +652,9 @@ class ModelToComponentFactory:
             OffsetIncrementModel: self.create_offset_increment,
             PageIncrementModel: self.create_page_increment,
             ParentStreamConfigModel: self.create_parent_stream_config,
+            PropertiesFromEndpointModel: self.create_properties_from_endpoint,
+            PropertyChunkingModel: self.create_property_chunking,
+            QueryPropertiesModel: self.create_query_properties,
             RecordFilterModel: self.create_record_filter,
             RecordSelectorModel: self.create_record_selector,
             RemoveFieldsModel: self.create_remove_fields,
@@ -2083,8 +2113,8 @@ class ModelToComponentFactory:
             parameters=model.parameters or {},
         )
 
+    @staticmethod
     def create_response_to_file_extractor(
-        self,
         model: ResponseToFileExtractorModel,
         **kwargs: Any,
     ) -> ResponseToFileExtractor:
@@ -2098,11 +2128,17 @@ class ModelToComponentFactory:
             factor=model.factor or 5, parameters=model.parameters or {}, config=config
         )
 
+    @staticmethod
+    def create_group_by_key(model: GroupByKeyMergeStrategyModel, config: Config) -> GroupByKey:
+        return GroupByKey(model.key, config=config, parameters=model.parameters or {})
+
     def create_http_requester(
         self,
         model: HttpRequesterModel,
         config: Config,
         decoder: Decoder = JsonDecoder(parameters={}),
+        query_properties_key: Optional[str] = None,
+        use_cache: Optional[bool] = None,
         *,
         name: str,
     ) -> HttpRequester:
@@ -2135,6 +2171,7 @@ class ModelToComponentFactory:
             request_body_json=model.request_body_json,
             request_headers=model.request_headers,
             request_parameters=model.request_parameters,
+            query_properties_key=query_properties_key,
             config=config,
             parameters=model.parameters or {},
         )
@@ -2142,7 +2179,7 @@ class ModelToComponentFactory:
         assert model.use_cache is not None  # for mypy
         assert model.http_method is not None  # for mypy
 
-        use_cache = model.use_cache and not self._disable_cache
+        should_use_cache = (model.use_cache or bool(use_cache)) and not self._disable_cache
 
         return HttpRequester(
             name=name,
@@ -2157,7 +2194,7 @@ class ModelToComponentFactory:
             disable_retries=self._disable_retries,
             parameters=model.parameters or {},
             message_repository=self._message_repository,
-            use_cache=use_cache,
+            use_cache=should_use_cache,
             decoder=decoder,
             stream_response=decoder.is_stream_response() if decoder else False,
         )
@@ -2261,10 +2298,11 @@ class ModelToComponentFactory:
         retriever = self._create_component_from_model(
             model=model.retriever,
             config=config,
-            name="",
+            name="dynamic_properties",
             primary_key=None,
             stream_slicer=combined_slicers,
             transformations=[],
+            use_cache=True,
         )
         schema_type_identifier = self._create_component_from_model(
             model.schema_type_identifier, config=config, parameters=model.parameters or {}
@@ -2602,6 +2640,79 @@ class ModelToComponentFactory:
             lazy_read_pointer=model_lazy_read_pointer,
         )
 
+    def create_properties_from_endpoint(
+        self, model: PropertiesFromEndpointModel, config: Config, **kwargs: Any
+    ) -> PropertiesFromEndpoint:
+        retriever = self._create_component_from_model(
+            model=model.retriever,
+            config=config,
+            name="dynamic_properties",
+            primary_key=None,
+            stream_slicer=None,
+            transformations=[],
+            use_cache=True,  # Enable caching on the HttpRequester/HttpClient because the properties endpoint will be called for every slice being processed, and it is highly unlikely for the response to different
+        )
+        return PropertiesFromEndpoint(
+            property_field_path=model.property_field_path,
+            retriever=retriever,
+            config=config,
+            parameters=model.parameters or {},
+        )
+
+    def create_property_chunking(
+        self, model: PropertyChunkingModel, config: Config, **kwargs: Any
+    ) -> PropertyChunking:
+        record_merge_strategy = (
+            self._create_component_from_model(
+                model=model.record_merge_strategy, config=config, **kwargs
+            )
+            if model.record_merge_strategy
+            else None
+        )
+
+        property_limit_type: PropertyLimitType
+        match model.property_limit_type:
+            case PropertyLimitTypeModel.property_count:
+                property_limit_type = PropertyLimitType.property_count
+            case PropertyLimitTypeModel.characters:
+                property_limit_type = PropertyLimitType.characters
+            case _:
+                raise ValueError(f"Invalid PropertyLimitType {property_limit_type}")
+
+        return PropertyChunking(
+            property_limit_type=property_limit_type,
+            property_limit=model.property_limit,
+            record_merge_strategy=record_merge_strategy,
+            config=config,
+            parameters=model.parameters or {},
+        )
+
+    def create_query_properties(
+        self, model: QueryPropertiesModel, config: Config, **kwargs: Any
+    ) -> QueryProperties:
+        if isinstance(model.property_list, list):
+            property_list = model.property_list
+        else:
+            property_list = self._create_component_from_model(
+                model=model.property_list, config=config, **kwargs
+            )
+
+        property_chunking = (
+            self._create_component_from_model(
+                model=model.property_chunking, config=config, **kwargs
+            )
+            if model.property_chunking
+            else None
+        )
+
+        return QueryProperties(
+            property_list=property_list,
+            always_include_properties=model.always_include_properties,
+            property_chunking=property_chunking,
+            config=config,
+            parameters=model.parameters or {},
+        )
+
     @staticmethod
     def create_record_filter(
         model: RecordFilterModel, config: Config, **kwargs: Any
@@ -2747,6 +2858,7 @@ class ModelToComponentFactory:
                 IncrementingCountCursorModel, DatetimeBasedCursorModel, CustomIncrementalSyncModel
             ]
         ] = None,
+        use_cache: Optional[bool] = None,
         **kwargs: Any,
     ) -> SimpleRetriever:
         decoder = (
@@ -2754,9 +2866,6 @@ class ModelToComponentFactory:
             if model.decoder
             else JsonDecoder(parameters={})
         )
-        requester = self._create_component_from_model(
-            model=model.requester, decoder=decoder, config=config, name=name
-        )
         record_selector = self._create_component_from_model(
             model=model.record_selector,
             name=name,
@@ -2765,6 +2874,57 @@ class ModelToComponentFactory:
             transformations=transformations,
             client_side_incremental_sync=client_side_incremental_sync,
         )
+
+        query_properties: Optional[QueryProperties] = None
+        query_properties_key: Optional[str] = None
+        if (
+            hasattr(model.requester, "request_parameters")
+            and model.requester.request_parameters
+            and isinstance(model.requester.request_parameters, Mapping)
+        ):
+            query_properties_definitions = []
+            for key, request_parameter in model.requester.request_parameters.items():
+                # When translating JSON schema into Pydantic models, enforcing types for arrays containing both
+                # concrete string complex object definitions like QueryProperties would get resolved to Union[str, Any].
+                # This adds the extra validation that we couldn't get for free in Pydantic model generation
+                if (
+                    isinstance(request_parameter, Mapping)
+                    and request_parameter.get("type") == "QueryProperties"
+                ):
+                    query_properties_key = key
+                    query_properties_definitions.append(request_parameter)
+                elif not isinstance(request_parameter, str):
+                    raise ValueError(
+                        f"Each element of request_parameters should be of type str or QueryProperties, but received {request_parameter.get('type')}"
+                    )
+
+            if len(query_properties_definitions) > 1:
+                raise ValueError(
+                    f"request_parameters only supports defining one QueryProperties field, but found {len(query_properties_definitions)} usages"
+                )
+
+            if len(query_properties_definitions) == 1:
+                query_properties = self.create_component(
+                    model_type=QueryPropertiesModel,
+                    component_definition=query_properties_definitions[0],
+                    config=config,
+                )
+
+            # Removes QueryProperties components from the interpolated mappings because it will be resolved in
+            # the provider from the slice directly instead of through jinja interpolation
+            if isinstance(model.requester.request_parameters, Mapping):
+                model.requester.request_parameters = self._remove_query_properties(
+                    model.requester.request_parameters
+                )
+
+        requester = self._create_component_from_model(
+            model=model.requester,
+            decoder=decoder,
+            name=name,
+            query_properties_key=query_properties_key,
+            use_cache=use_cache,
+            config=config,
+        )
         url_base = (
             model.requester.url_base
             if hasattr(model.requester, "url_base")
@@ -2870,9 +3030,21 @@ class ModelToComponentFactory:
             cursor=cursor,
             config=config,
             ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests,
+            additional_query_properties=query_properties,
             parameters=model.parameters or {},
         )
 
+    @staticmethod
+    def _remove_query_properties(
+        request_parameters: Mapping[str, Union[Any, str]],
+    ) -> Mapping[str, Union[Any, str]]:
+        return {
+            parameter_field: request_parameter
+            for parameter_field, request_parameter in request_parameters.items()
+            if not isinstance(request_parameter, Mapping)
+            or not request_parameter.get("type") == "QueryProperties"
+        }
+
     def create_state_delegating_stream(
         self,
         model: StateDelegatingStreamModel,

airbyte_cdk-6.45.0/airbyte_cdk/sources/declarative/requesters/query_properties/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+
+from airbyte_cdk.sources.declarative.requesters.query_properties.properties_from_endpoint import (
+    PropertiesFromEndpoint,
+)
+from airbyte_cdk.sources.declarative.requesters.query_properties.property_chunking import (
+    PropertyChunking,
+)
+from airbyte_cdk.sources.declarative.requesters.query_properties.query_properties import (
+    QueryProperties,
+)
+
+__all__ = ["PropertiesFromEndpoint", "PropertyChunking", "QueryProperties"]

airbyte_cdk-6.45.0/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Iterable, List, Mapping, Optional
+
+import dpath
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.declarative.retrievers import Retriever
+from airbyte_cdk.sources.types import Config, StreamSlice
+
+
+@dataclass
+class PropertiesFromEndpoint:
+    """
+    Component that defines the behavior around how to dynamically retrieve a set of request properties from an
+    API endpoint. The set retrieved can then be injected into the requests to extract records from an API source.
+    """
+
+    property_field_path: List[str]
+    retriever: Retriever
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._property_field_path = [
+            InterpolatedString(string=property_field, parameters=parameters)
+            for property_field in self.property_field_path
+        ]
+
+    def get_properties_from_endpoint(self, stream_slice: Optional[StreamSlice]) -> Iterable[str]:
+        response_properties = self.retriever.read_records(
+            records_schema={}, stream_slice=stream_slice
+        )
+        for property_obj in response_properties:
+            path = [
+                node.eval(self.config) if not isinstance(node, str) else node
+                for node in self._property_field_path
+            ]
+            yield dpath.get(property_obj, path, default=[])  # type: ignore # extracted will be a MutableMapping, given input data structure