airbyte-cdk 6.45.0.dev4107__py3-none-any.whl → 6.45.0.post20.dev14369762306__py3-none-any.whl

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+
 # generated by datamodel-codegen:
 #   filename:  declarative_component_schema.yaml
 
@@ -42,13 +44,15 @@ class BearerAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class CheckStream(BaseModel):
-    type: Literal["CheckStream"]
-    stream_names: List[str] = Field(
-        ...,
-        description="Names of the streams to try reading from when running a check operation.",
-        examples=[["users"], ["users", "contacts"]],
-        title="Stream Names",
+class DynamicStreamCheckConfig(BaseModel):
+    type: Literal["DynamicStreamCheckConfig"]
+    dynamic_stream_name: str = Field(
+        ..., description="The dynamic stream name.", title="Dynamic Stream Name"
+    )
+    stream_count: Optional[int] = Field(
+        0,
+        description="The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.",
+        title="Stream Count",
     )
 
 
@@ -716,6 +720,17 @@ class ExponentialBackoffStrategy(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class GroupByKeyMergeStrategy(BaseModel):
+    type: Literal["GroupByKeyMergeStrategy"]
+    key: Union[str, List[str]] = Field(
+        ...,
+        description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.",
+        examples=["id", ["parent_id", "end_date"]],
+        title="Key",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class SessionTokenRequestBearerAuthenticator(BaseModel):
     type: Literal["Bearer"]
 
@@ -1187,6 +1202,31 @@ class PrimaryKey(BaseModel):
     )
 
 
+class PropertyLimitType(Enum):
+    characters = "characters"
+    property_count = "property_count"
+
+
+class PropertyChunking(BaseModel):
+    type: Literal["PropertyChunking"]
+    property_limit_type: PropertyLimitType = Field(
+        ...,
+        description="The type used to determine the maximum number of properties per chunk",
+        title="Property Limit Type",
+    )
+    property_limit: Optional[int] = Field(
+        None,
+        description="The maximum amount of properties that can be retrieved per request according to the limit type.",
+        title="Property Limit",
+    )
+    record_merge_strategy: Optional[GroupByKeyMergeStrategy] = Field(
+        None,
+        description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination",
+        title="Record Merge Strategy",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class RecordFilter(BaseModel):
     type: Literal["RecordFilter"]
     condition: Optional[str] = Field(
@@ -1523,6 +1563,17 @@ class AuthFlow(BaseModel):
     oauth_config_specification: Optional[OAuthConfigSpecification] = None
 
 
+class CheckStream(BaseModel):
+    type: Literal["CheckStream"]
+    stream_names: Optional[List[str]] = Field(
+        None,
+        description="Names of the streams to try reading from when running a check operation.",
+        examples=[["users"], ["users", "contacts"]],
+        title="Stream Names",
+    )
+    dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
+
+
 class IncrementingCountCursor(BaseModel):
     type: Literal["IncrementingCountCursor"]
     cursor_field: str = Field(
@@ -1890,9 +1941,10 @@ class DeclarativeSource1(BaseModel):
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
     api_budget: Optional[HTTPAPIBudget] = None
-    max_concurrent_async_job_count: Optional[int] = Field(
+    max_concurrent_async_job_count: Optional[Union[int, str]] = Field(
         None,
         description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.",
+        examples=[3, "{{ config['max_concurrent_async_job_count'] }}"],
         title="Maximum Concurrent Asynchronous Jobs",
     )
     metadata: Optional[Dict[str, Any]] = Field(
@@ -1922,9 +1974,10 @@ class DeclarativeSource2(BaseModel):
     spec: Optional[Spec] = None
     concurrency_level: Optional[ConcurrencyLevel] = None
     api_budget: Optional[HTTPAPIBudget] = None
-    max_concurrent_async_job_count: Optional[int] = Field(
+    max_concurrent_async_job_count: Optional[Union[int, str]] = Field(
         None,
         description="Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.",
+        examples=[3, "{{ config['max_concurrent_async_job_count'] }}"],
         title="Maximum Concurrent Asynchronous Jobs",
     )
     metadata: Optional[Dict[str, Any]] = Field(
@@ -1989,31 +2042,6 @@ class SelectiveAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class FileUploader(BaseModel):
-    type: Literal["FileUploader"]
-    requester: Union[CustomRequester, HttpRequester] = Field(
-        ...,
-        description="Requester component that describes how to prepare HTTP requests to send to the source API.",
-    )
-    download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
-        ...,
-        description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response",
-    )
-    file_extractor: Optional[Union[CustomRecordExtractor, DpathExtractor]] = Field(
-        None,
-        description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content",
-    )
-    filename_extractor: Optional[str] = Field(
-        None,
-        description="Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.",
-        examples=[
-            "{{ record.id }}/{{ record.file_name }}/",
-            "{{ record.id }}_{{ record.file_name }}/",
-        ],
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class DeclarativeStream(BaseModel):
     class Config:
         extra = Extra.allow
@@ -2072,11 +2100,6 @@ class DeclarativeStream(BaseModel):
         description="Array of state migrations to be applied on the input state",
         title="State Migrations",
     )
-    file_uploader: Optional[FileUploader] = Field(
-        None,
-        description="(experimental) Describes how to fetch a file",
-        title="File Uploader",
-    )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
@@ -2202,7 +2225,7 @@ class HttpRequester(BaseModel):
         examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
         title="Request Headers",
     )
-    request_parameters: Optional[Union[str, Dict[str, str]]] = Field(
+    request_parameters: Optional[Union[str, Dict[str, Union[str, Any]]]] = Field(
         None,
         description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
         examples=[
@@ -2292,6 +2315,40 @@ class ParentStreamConfig(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class PropertiesFromEndpoint(BaseModel):
+    type: Literal["PropertiesFromEndpoint"]
+    property_field_path: List[str] = Field(
+        ...,
+        description="Describes the path to the field that should be extracted",
+        examples=[["name"]],
+    )
+    retriever: Union[CustomRetriever, SimpleRetriever] = Field(
+        ...,
+        description="Requester component that describes how to fetch the properties to query from a remote API endpoint.",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
+class QueryProperties(BaseModel):
+    type: Literal["QueryProperties"]
+    property_list: Union[List[str], PropertiesFromEndpoint] = Field(
+        ...,
+        description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint",
+        title="Property List",
+    )
+    always_include_properties: Optional[List[str]] = Field(
+        None,
+        description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.",
+        title="Always Include Properties",
+    )
+    property_chunking: Optional[PropertyChunking] = Field(
+        None,
+        description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.",
+        title="Property Chunking",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class StateDelegatingStream(BaseModel):
     type: Literal["StateDelegatingStream"]
     name: str = Field(..., description="The stream name.", example=["Users"], title="Name")
@@ -2331,7 +2388,15 @@ class SimpleRetriever(BaseModel):
             CustomPartitionRouter,
             ListPartitionRouter,
             SubstreamPartitionRouter,
-            List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
+            GroupingPartitionRouter,
+            List[
+                Union[
+                    CustomPartitionRouter,
+                    ListPartitionRouter,
+                    SubstreamPartitionRouter,
+                    GroupingPartitionRouter,
+                ]
+            ],
         ]
     ] = Field(
         [],
@@ -2413,7 +2478,15 @@ class AsyncRetriever(BaseModel):
             CustomPartitionRouter,
             ListPartitionRouter,
             SubstreamPartitionRouter,
-            List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
+            GroupingPartitionRouter,
+            List[
+                Union[
+                    CustomPartitionRouter,
+                    ListPartitionRouter,
+                    SubstreamPartitionRouter,
+                    GroupingPartitionRouter,
+                ]
+            ],
         ]
     ] = Field(
         [],
@@ -2465,6 +2538,29 @@ class SubstreamPartitionRouter(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class GroupingPartitionRouter(BaseModel):
+    type: Literal["GroupingPartitionRouter"]
+    group_size: int = Field(
+        ...,
+        description="The number of partitions to include in each group. This determines how many partition values are batched together in a single slice.",
+        examples=[10, 50],
+        title="Group Size",
+    )
+    underlying_partition_router: Union[
+        CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
+    ] = Field(
+        ...,
+        description="The partition router whose output will be grouped. This can be any valid partition router component.",
+        title="Underlying Partition Router",
+    )
+    deduplicate: Optional[bool] = Field(
+        True,
+        description="If true, ensures that partitions are unique within each group by removing duplicates based on the partition key.",
+        title="Deduplicate Partitions",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class HttpComponentsResolver(BaseModel):
     type: Literal["HttpComponentsResolver"]
     retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
@@ -2478,6 +2574,9 @@ class HttpComponentsResolver(BaseModel):
 
 class DynamicDeclarativeStream(BaseModel):
     type: Literal["DynamicDeclarativeStream"]
+    name: Optional[str] = Field(
+        "", description="The dynamic stream name.", example=["Tables"], title="Name"
+    )
     stream_template: DeclarativeStream = Field(
         ..., description="Reference to the stream template.", title="Stream Template"
     )
@@ -2494,10 +2593,10 @@ CompositeErrorHandler.update_forward_refs()
 DeclarativeSource1.update_forward_refs()
 DeclarativeSource2.update_forward_refs()
 SelectiveAuthenticator.update_forward_refs()
-FileUploader.update_forward_refs()
 DeclarativeStream.update_forward_refs()
 SessionTokenAuthenticator.update_forward_refs()
 DynamicSchemaLoader.update_forward_refs()
 ParentStreamConfig.update_forward_refs()
+PropertiesFromEndpoint.update_forward_refs()
 SimpleRetriever.update_forward_refs()
 AsyncRetriever.update_forward_refs()

airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py

@@ -4,7 +4,7 @@
 
 import copy
 import typing
-from typing import Any, Mapping
+from typing import Any, Mapping, Optional
 
 PARAMETERS_STR = "$parameters"
 
@@ -94,6 +94,7 @@ class ManifestComponentTransformer:
         parent_field_identifier: str,
         declarative_component: Mapping[str, Any],
         parent_parameters: Mapping[str, Any],
+        use_parent_parameters: Optional[bool] = None,
     ) -> Mapping[str, Any]:
         """
         Recursively transforms the specified declarative component and subcomponents to propagate parameters and insert the
@@ -103,6 +104,7 @@ class ManifestComponentTransformer:
         :param declarative_component: The current component that is having type and parameters added
         :param parent_field_identifier: The name of the field of the current component coming from the parent component
         :param parent_parameters: The parameters set on parent components defined before the current component
+        :param use_parent_parameters: If set, parent parameters will be used as the source of truth when key names are the same
         :return: A deep copy of the transformed component with types and parameters persisted to it
         """
         propagated_component = dict(copy.deepcopy(declarative_component))
@@ -130,7 +132,11 @@ class ManifestComponentTransformer:
         # level take precedence
         current_parameters = dict(copy.deepcopy(parent_parameters))
         component_parameters = propagated_component.pop(PARAMETERS_STR, {})
-        current_parameters = {**current_parameters, **component_parameters}
+        current_parameters = (
+            {**component_parameters, **current_parameters}
+            if use_parent_parameters
+            else {**current_parameters, **component_parameters}
+        )
 
         # Parameters should be applied to the current component fields with the existing field taking precedence over parameters if
         # both exist
@@ -145,7 +151,10 @@ class ManifestComponentTransformer:
                 excluded_parameter = current_parameters.pop(field_name, None)
                 parent_type_field_identifier = f"{propagated_component.get('type')}.{field_name}"
                 propagated_component[field_name] = self.propagate_types_and_parameters(
-                    parent_type_field_identifier, field_value, current_parameters
+                    parent_type_field_identifier,
+                    field_value,
+                    current_parameters,
+                    use_parent_parameters=use_parent_parameters,
                 )
                 if excluded_parameter:
                     current_parameters[field_name] = excluded_parameter
@@ -158,7 +167,10 @@ class ManifestComponentTransformer:
                             f"{propagated_component.get('type')}.{field_name}"
                         )
                         field_value[i] = self.propagate_types_and_parameters(
-                            parent_type_field_identifier, element, current_parameters
+                            parent_type_field_identifier,
+                            element,
+                            current_parameters,
+                            use_parent_parameters=use_parent_parameters,
                         )
                 if excluded_parameter:
                     current_parameters[field_name] = excluded_parameter