airbyte-cdk 6.45.10__py3-none-any.whl → 6.46.0.post4.dev14712712756__py3-none-any.whl

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -417,15 +417,18 @@ class JwtAuthenticator(BaseModel):
         ...,
         description="Secret used to sign the JSON web token.",
         examples=["{{ config['secret_key'] }}"],
+        title="Secret Key",
     )
     base64_encode_secret_key: Optional[bool] = Field(
         False,
         description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.',
+        title="Base64-encode Secret Key",
     )
     algorithm: Algorithm = Field(
         ...,
         description="Algorithm used to sign the JSON web token.",
         examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"],
+        title="Algorithm",
     )
     token_duration: Optional[int] = Field(
         1200,
@@ -880,20 +883,17 @@ class FlattenFields(BaseModel):
 
 
 class KeyTransformation(BaseModel):
-    prefix: Optional[Union[str, None]] = Field(
+    type: Literal["KeyTransformation"]
+    prefix: Optional[str] = Field(
         None,
         description="Prefix to add for object keys. If not provided original keys remain unchanged.",
-        examples=[
-            "flattened_",
-        ],
+        examples=["flattened_"],
         title="Key Prefix",
     )
-    suffix: Optional[Union[str, None]] = Field(
+    suffix: Optional[str] = Field(
         None,
         description="Suffix to add for object keys. If not provided original keys remain unchanged.",
-        examples=[
-            "_flattened",
-        ],
+        examples=["_flattened"],
         title="Key Suffix",
     )
 
@@ -916,7 +916,7 @@ class DpathFlattenFields(BaseModel):
         description="Whether to replace the origin record or not. Default is False.",
         title="Replace Origin Record",
     )
-    key_transformation: Optional[Union[KeyTransformation, None]] = Field(
+    key_transformation: Optional[KeyTransformation] = Field(
         None,
         description="Transformation for object keys. If not provided, original key will be used.",
         title="Key transformation",
@@ -1260,13 +1260,14 @@ class RecordFilter(BaseModel):
             "{{ record['created_at'] >= stream_interval['start_time'] }}",
             "{{ record.status in ['active', 'expired'] }}",
         ],
+        title="Condition",
     )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
 class SchemaNormalization(Enum):
-    None_ = "None"
     Default = "Default"
+    None_ = "None"
 
 
 class RemoveFields(BaseModel):
@@ -1802,14 +1803,18 @@ class DefaultErrorHandler(BaseModel):
 class DefaultPaginator(BaseModel):
     type: Literal["DefaultPaginator"]
     pagination_strategy: Union[
-        CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement
+        PageIncrement, OffsetIncrement, CursorPagination, CustomPaginationStrategy
     ] = Field(
         ...,
         description="Strategy defining how records are paginated.",
         title="Pagination Strategy",
     )
-    page_size_option: Optional[RequestOption] = None
-    page_token_option: Optional[Union[RequestOption, RequestPath]] = None
+    page_size_option: Optional[RequestOption] = Field(
+        None, title="Inject Page Size Into Outgoing HTTP Request"
+    )
+    page_token_option: Optional[Union[RequestOption, RequestPath]] = Field(
+        None, title="Inject Page Token Into Outgoing HTTP Request"
+    )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
@@ -1850,20 +1855,21 @@ class ListPartitionRouter(BaseModel):
 
 class RecordSelector(BaseModel):
     type: Literal["RecordSelector"]
-    extractor: Union[CustomRecordExtractor, DpathExtractor]
-    record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field(
+    extractor: Union[DpathExtractor, CustomRecordExtractor]
+    record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field(
         None,
         description="Responsible for filtering records to be emitted by the Source.",
         title="Record Filter",
     )
     schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field(
-        SchemaNormalization.None_,
+        None,
         description="Responsible for normalization according to the schema.",
         title="Schema Normalization",
     )
     transform_before_filtering: Optional[bool] = Field(
-        False,
+        None,
         description="If true, transformation will be applied before record filtering.",
+        title="Transform Before Filtering",
     )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
@@ -2096,26 +2102,26 @@ class DeclarativeStream(BaseModel):
         extra = Extra.allow
 
     type: Literal["DeclarativeStream"]
-    retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
+    name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name")
+    retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field(
         ...,
         description="Component used to coordinate how records are extracted across stream slices and request pages.",
         title="Retriever",
     )
     incremental_sync: Optional[
-        Union[CustomIncrementalSync, DatetimeBasedCursor, IncrementingCountCursor]
+        Union[DatetimeBasedCursor, IncrementingCountCursor, CustomIncrementalSync]
     ] = Field(
         None,
         description="Component used to fetch data incrementally based on a time field in the data.",
         title="Incremental Sync",
     )
-    name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name")
     primary_key: Optional[PrimaryKey] = Field(
         "", description="The primary key of the stream.", title="Primary Key"
     )
     schema_loader: Optional[
         Union[
-            DynamicSchemaLoader,
             InlineSchemaLoader,
+            DynamicSchemaLoader,
             JsonFileSchemaLoader,
             CustomSchemaLoader,
         ]
@@ -2205,7 +2211,7 @@ class HttpRequester(BaseModel):
     type: Literal["HttpRequester"]
     url_base: str = Field(
         ...,
-        description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
+        description="The Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
         examples=[
             "https://connect.squareup.com/v2",
             "{{ config['base_url'] or 'https://app.posthog.com'}}/api",
@@ -2216,7 +2222,7 @@ class HttpRequester(BaseModel):
     )
     path: Optional[str] = Field(
         None,
-        description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
+        description="The Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
         examples=[
             "/products",
             "/quotes/{{ stream_partition['id'] }}/quote_line_groups",
@@ -2224,38 +2230,36 @@ class HttpRequester(BaseModel):
         ],
         title="URL Path",
     )
+    http_method: Optional[HttpMethod] = Field(
+        HttpMethod.GET,
+        description="The HTTP method used to fetch data from the source (can be GET or POST).",
+        examples=["GET", "POST"],
+        title="HTTP Method",
+    )
     authenticator: Optional[
         Union[
+            NoAuth,
             ApiKeyAuthenticator,
             BasicHttpAuthenticator,
             BearerAuthenticator,
-            CustomAuthenticator,
             OAuthAuthenticator,
             JwtAuthenticator,
-            NoAuth,
             SessionTokenAuthenticator,
-            LegacySessionTokenAuthenticator,
             SelectiveAuthenticator,
+            CustomAuthenticator,
+            LegacySessionTokenAuthenticator,
         ]
     ] = Field(
         None,
         description="Authentication method to use for requests sent to the API.",
         title="Authenticator",
     )
-    error_handler: Optional[
-        Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]
-    ] = Field(
+    fetch_properties_from_endpoint: Optional[PropertiesFromEndpoint] = Field(
         None,
-        description="Error handler component that defines how to handle errors.",
-        title="Error Handler",
-    )
-    http_method: Optional[HttpMethod] = Field(
-        HttpMethod.GET,
-        description="The HTTP method used to fetch data from the source (can be GET or POST).",
-        examples=["GET", "POST"],
-        title="HTTP Method",
+        description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.",
+        title="Fetch Properties from Endpoint",
     )
-    request_body_data: Optional[Union[str, Dict[str, str]]] = Field(
+    request_body_data: Optional[Union[Dict[str, str], str]] = Field(
         None,
         description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.",
         examples=[
@@ -2263,7 +2267,7 @@ class HttpRequester(BaseModel):
         ],
         title="Request Body Payload (Non-JSON)",
     )
-    request_body_json: Optional[Union[str, Dict[str, Any]]] = Field(
+    request_body_json: Optional[Union[Dict[str, Any], str]] = Field(
         None,
         description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.",
         examples=[
@@ -2273,13 +2277,13 @@ class HttpRequester(BaseModel):
         ],
         title="Request Body JSON Payload",
     )
-    request_headers: Optional[Union[str, Dict[str, str]]] = Field(
+    request_headers: Optional[Union[Dict[str, str], str]] = Field(
         None,
         description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.",
         examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
         title="Request Headers",
     )
-    request_parameters: Optional[Union[str, Dict[str, Union[str, Any]]]] = Field(
+    request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field(
         None,
         description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
         examples=[
@@ -2292,6 +2296,13 @@ class HttpRequester(BaseModel):
         ],
         title="Query Parameters",
     )
+    error_handler: Optional[
+        Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler]
+    ] = Field(
+        None,
+        description="Error handler component that defines how to handle errors.",
+        title="Error Handler",
+    )
     use_cache: Optional[bool] = Field(
         False,
         description="Enables stream requests caching. This field is automatically set by the CDK.",
@@ -2409,25 +2420,41 @@ class StateDelegatingStream(BaseModel):
     full_refresh_stream: DeclarativeStream = Field(
         ...,
         description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.",
-        title="Retriever",
+        title="Full Refresh Stream",
     )
     incremental_stream: DeclarativeStream = Field(
         ...,
         description="Component used to coordinate how records are extracted across stream slices and request pages when the state provided.",
-        title="Retriever",
+        title="Incremental Stream",
     )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
 class SimpleRetriever(BaseModel):
     type: Literal["SimpleRetriever"]
-    record_selector: RecordSelector = Field(
+    requester: Union[HttpRequester, CustomRequester] = Field(
         ...,
-        description="Component that describes how to extract records from a HTTP response.",
+        description="Requester component that describes how to prepare HTTP requests to send to the source API.",
     )
-    requester: Union[CustomRequester, HttpRequester] = Field(
+    decoder: Optional[
+        Union[
+            CsvDecoder,
+            GzipDecoder,
+            JsonDecoder,
+            JsonlDecoder,
+            IterableDecoder,
+            XmlDecoder,
+            ZipfileDecoder,
+            CustomDecoder,
+        ]
+    ] = Field(
+        None,
+        description="Component decoding the response so records can be extracted.",
+        title="Decoder",
+    )
+    record_selector: RecordSelector = Field(
         ...,
-        description="Requester component that describes how to prepare HTTP requests to send to the source API.",
+        description="Component that describes how to extract records from a HTTP response.",
     )
     paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
         None,
@@ -2439,16 +2466,16 @@ class SimpleRetriever(BaseModel):
     )
     partition_router: Optional[
         Union[
-            CustomPartitionRouter,
             ListPartitionRouter,
             SubstreamPartitionRouter,
             GroupingPartitionRouter,
+            CustomPartitionRouter,
             List[
                 Union[
-                    CustomPartitionRouter,
                     ListPartitionRouter,
                     SubstreamPartitionRouter,
                     GroupingPartitionRouter,
+                    CustomPartitionRouter,
                 ]
             ],
         ]
@@ -2457,22 +2484,6 @@ class SimpleRetriever(BaseModel):
         description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
         title="Partition Router",
     )
-    decoder: Optional[
-        Union[
-            CustomDecoder,
-            CsvDecoder,
-            GzipDecoder,
-            JsonDecoder,
-            JsonlDecoder,
-            IterableDecoder,
-            XmlDecoder,
-            ZipfileDecoder,
-        ]
-    ] = Field(
-        None,
-        description="Component decoding the response so records can be extracted.",
-        title="Decoder",
-    )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
@@ -2485,21 +2496,21 @@ class AsyncRetriever(BaseModel):
     status_mapping: AsyncJobStatusMap = Field(
         ..., description="Async Job Status to Airbyte CDK Async Job Status mapping."
     )
-    status_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
+    status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field(
         ..., description="Responsible for fetching the actual status of the async job."
     )
-    download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
+    download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field(
         ...,
         description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.",
     )
     download_extractor: Optional[
-        Union[CustomRecordExtractor, DpathExtractor, ResponseToFileExtractor]
+        Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor]
     ] = Field(None, description="Responsible for fetching the records from provided urls.")
-    creation_requester: Union[CustomRequester, HttpRequester] = Field(
+    creation_requester: Union[HttpRequester, CustomRequester] = Field(
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.",
     )
-    polling_requester: Union[CustomRequester, HttpRequester] = Field(
+    polling_requester: Union[HttpRequester, CustomRequester] = Field(
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.",
     )
@@ -2507,11 +2518,11 @@ class AsyncRetriever(BaseModel):
         None,
         description="The time in minutes after which the single Async Job should be considered as Timed Out.",
     )
-    download_target_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
+    download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field(
         None,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.",
     )
-    download_requester: Union[CustomRequester, HttpRequester] = Field(
+    download_requester: Union[HttpRequester, CustomRequester] = Field(
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.",
     )
@@ -2519,26 +2530,26 @@ class AsyncRetriever(BaseModel):
         None,
         description="Paginator component that describes how to navigate through the API's pages during download.",
     )
-    abort_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
+    abort_requester: Optional[Union[HttpRequester, CustomRequester]] = Field(
         None,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.",
     )
-    delete_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
+    delete_requester: Optional[Union[HttpRequester, CustomRequester]] = Field(
         None,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.",
     )
     partition_router: Optional[
         Union[
-            CustomPartitionRouter,
             ListPartitionRouter,
             SubstreamPartitionRouter,
             GroupingPartitionRouter,
+            CustomPartitionRouter,
             List[
                 Union[
-                    CustomPartitionRouter,
                     ListPartitionRouter,
                     SubstreamPartitionRouter,
                     GroupingPartitionRouter,
+                    CustomPartitionRouter,
                 ]
             ],
         ]
@@ -2549,7 +2560,6 @@ class AsyncRetriever(BaseModel):
     )
     decoder: Optional[
         Union[
-            CustomDecoder,
             CsvDecoder,
             GzipDecoder,
             JsonDecoder,
@@ -2557,6 +2567,7 @@ class AsyncRetriever(BaseModel):
             IterableDecoder,
             XmlDecoder,
             ZipfileDecoder,
+            CustomDecoder,
         ]
     ] = Field(
         None,
@@ -2565,7 +2576,6 @@ class AsyncRetriever(BaseModel):
     )
     download_decoder: Optional[
         Union[
-            CustomDecoder,
             CsvDecoder,
             GzipDecoder,
             JsonDecoder,
@@ -2573,6 +2583,7 @@ class AsyncRetriever(BaseModel):
             IterableDecoder,
             XmlDecoder,
             ZipfileDecoder,
+            CustomDecoder,
         ]
     ] = Field(
         None,
@@ -2650,6 +2661,7 @@ SelectiveAuthenticator.update_forward_refs()
 FileUploader.update_forward_refs()
 DeclarativeStream.update_forward_refs()
 SessionTokenAuthenticator.update_forward_refs()
+HttpRequester.update_forward_refs()
 DynamicSchemaLoader.update_forward_refs()
 ParentStreamConfig.update_forward_refs()
 PropertiesFromEndpoint.update_forward_refs()

airbyte_cdk/sources/declarative/parsers/custom_exceptions.py

@@ -19,3 +19,12 @@ class UndefinedReferenceException(Exception):
 
     def __init__(self, path: str, reference: str) -> None:
         super().__init__(f"Undefined reference {reference} from {path}")
+
+
+class ManifestNormalizationException(Exception):
+    """
+    Raised when a circular reference is detected in a manifest.
+    """
+
+    def __init__(self, message: str) -> None:
+        super().__init__(f"Failed to deduplicate manifest: {message}")

airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py

@@ -4,7 +4,7 @@
 
 import copy
 import typing
-from typing import Any, Mapping, Optional
+from typing import Any, Dict, Mapping, Optional
 
 PARAMETERS_STR = "$parameters"
 
@@ -95,7 +95,7 @@ class ManifestComponentTransformer:
         declarative_component: Mapping[str, Any],
         parent_parameters: Mapping[str, Any],
         use_parent_parameters: Optional[bool] = None,
-    ) -> Mapping[str, Any]:
+    ) -> Dict[str, Any]:
         """
         Recursively transforms the specified declarative component and subcomponents to propagate parameters and insert the
         default component type if it was not already present. The resulting transformed components are a deep copy of the input