airbyte-cdk 6.20.2.dev0__py3-none-any.whl → 6.21.0__py3-none-any.whl

airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -7,9 +7,12 @@ from dataclasses import dataclass
 from io import BufferedIOBase, TextIOWrapper
 from typing import Any, Generator, MutableMapping, Optional
 
+import orjson
 import requests
 
+from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
+from airbyte_cdk.utils import AirbyteTracedException
 
 logger = logging.getLogger("airbyte")
 
@@ -42,6 +45,46 @@ class GzipParser(Parser):
             yield from self.inner_parser.parse(gzipobj)
 
 
+@dataclass
+class JsonParser(Parser):
+    encoding: str = "utf-8"
+
+    def parse(self, data: BufferedIOBase) -> Generator[MutableMapping[str, Any], None, None]:
+        """
+        Attempts to deserialize data using orjson library. As an extra layer of safety we fallback on the json library to deserialize the data.
+        """
+        raw_data = data.read()
+        body_json = self._parse_orjson(raw_data) or self._parse_json(raw_data)
+
+        if body_json is None:
+            raise AirbyteTracedException(
+                message="Response JSON data failed to be parsed. See logs for more information.",
+                internal_message=f"Response JSON data failed to be parsed.",
+                failure_type=FailureType.system_error,
+            )
+
+        if isinstance(body_json, list):
+            yield from body_json
+        else:
+            yield from [body_json]
+
+    def _parse_orjson(self, raw_data: bytes) -> Optional[Any]:
+        try:
+            return orjson.loads(raw_data.decode(self.encoding))
+        except Exception as exc:
+            logger.debug(
+                f"Failed to parse JSON data using orjson library. Falling back to json library. {exc}"
+            )
+            return None
+
+    def _parse_json(self, raw_data: bytes) -> Optional[Any]:
+        try:
+            return json.loads(raw_data.decode(self.encoding))
+        except Exception as exc:
+            logger.error(f"Failed to parse JSON data using json library. {exc}")
+            return None
+
+
 @dataclass
 class JsonLineParser(Parser):
     encoding: Optional[str] = "utf-8"

airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py

@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+import zipfile
+from dataclasses import dataclass
+from io import BytesIO
+from typing import Any, Generator, MutableMapping
+
+import orjson
+import requests
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.declarative.decoders import Decoder
+from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import (
+    Parser,
+)
+from airbyte_cdk.utils import AirbyteTracedException
+
+logger = logging.getLogger("airbyte")
+
+
+@dataclass
+class ZipfileDecoder(Decoder):
+    parser: Parser
+
+    def is_stream_response(self) -> bool:
+        return False
+
+    def decode(
+        self, response: requests.Response
+    ) -> Generator[MutableMapping[str, Any], None, None]:
+        try:
+            with zipfile.ZipFile(BytesIO(response.content)) as zip_file:
+                for file_name in zip_file.namelist():
+                    unzipped_content = zip_file.read(file_name)
+                    buffered_content = BytesIO(unzipped_content)
+                    try:
+                        yield from self.parser.parse(buffered_content)
+                    except Exception as e:
+                        logger.error(
+                            f"Failed to parse file: {file_name} from zip file: {response.request.url} with exception {e}."
+                        )
+                        raise AirbyteTracedException(
+                            message=f"Failed to parse file: {file_name} from zip file.",
+                            internal_message=f"Failed to parse file: {file_name} from zip file: {response.request.url}.",
+                            failure_type=FailureType.system_error,
+                        ) from e
+        except zipfile.BadZipFile as e:
+            logger.error(
+                f"Received an invalid zip file in response to URL: {response.request.url}. "
+                f"The size of the response body is: {len(response.content)}"
+            )
+            raise AirbyteTracedException(
+                message="Received an invalid zip file in response.",
+                internal_message=f"Received an invalid zip file in response to URL: {response.request.url}.",
+                failure_type=FailureType.system_error,
+            ) from e

airbyte_cdk/sources/declarative/extractors/record_filter.py

@@ -59,11 +59,13 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
 
     def __init__(
         self,
-        cursor: Union[DatetimeBasedCursor, PerPartitionWithGlobalCursor, GlobalSubstreamCursor],
+        date_time_based_cursor: DatetimeBasedCursor,
+        substream_cursor: Optional[Union[PerPartitionWithGlobalCursor, GlobalSubstreamCursor]],
         **kwargs: Any,
     ):
         super().__init__(**kwargs)
-        self._cursor = cursor
+        self._date_time_based_cursor = date_time_based_cursor
+        self._substream_cursor = substream_cursor
 
     def filter_records(
         self,
@@ -75,7 +77,7 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
         records = (
             record
             for record in records
-            if self._cursor.should_be_synced(
+            if (self._substream_cursor or self._date_time_based_cursor).should_be_synced(
                 # Record is created on the fly to align with cursors interface; stream name is ignored as we don't need it here
                 # Record stream name is empty cause it is not used durig the filtering
                 Record(data=record, associated_slice=stream_slice, stream_name="")

airbyte_cdk/sources/declarative/incremental/__init__.py

@@ -2,10 +2,6 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.incremental.concurrent_partition_cursor import (
-    ConcurrentCursorFactory,
-    ConcurrentPerPartitionCursor,
-)
 from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
 from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
 from airbyte_cdk.sources.declarative.incremental.global_substream_cursor import (
@@ -25,8 +21,6 @@ from airbyte_cdk.sources.declarative.incremental.resumable_full_refresh_cursor i
 
 __all__ = [
     "CursorFactory",
-    "ConcurrentCursorFactory",
-    "ConcurrentPerPartitionCursor",
     "DatetimeBasedCursor",
     "DeclarativeCursor",
     "GlobalSubstreamCursor",

airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py

@@ -64,9 +64,6 @@ class Timer:
         else:
             raise RuntimeError("Global substream cursor timer not started")
 
-    def is_running(self) -> bool:
-        return self._start is not None
-
 
 class GlobalSubstreamCursor(DeclarativeCursor):
     """

airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py

@@ -303,21 +303,6 @@ class PerPartitionCursor(DeclarativeCursor):
             raise ValueError("A partition needs to be provided in order to get request body json")
 
     def should_be_synced(self, record: Record) -> bool:
-        if (
-            record.associated_slice
-            and self._to_partition_key(record.associated_slice.partition)
-            not in self._cursor_per_partition
-        ):
-            partition_state = (
-                self._state_to_migrate_from
-                if self._state_to_migrate_from
-                else self._NO_CURSOR_STATE
-            )
-            cursor = self._create_cursor(partition_state)
-
-            self._cursor_per_partition[
-                self._to_partition_key(record.associated_slice.partition)
-            ] = cursor
         return self._get_cursor(record).should_be_synced(
             self._convert_record_to_cursor_record(record)
         )

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -22,6 +22,7 @@ from airbyte_cdk.models import (
     ConnectorSpecification,
     FailureType,
 )
+from airbyte_cdk.sources.declarative.checks import COMPONENTS_CHECKER_TYPE_MAPPING
 from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
@@ -107,7 +108,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         if "type" not in check:
             check["type"] = "CheckStream"
         check_stream = self._constructor.create_component(
-            CheckStreamModel,
+            COMPONENTS_CHECKER_TYPE_MAPPING[check["type"]],
             check,
             dict(),
             emit_connector_builder_messages=self._emit_connector_builder_messages,

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -52,6 +52,15 @@ class CheckStream(BaseModel):
     )
 
 
+class CheckDynamicStream(BaseModel):
+    type: Literal["CheckDynamicStream"]
+    stream_count: int = Field(
+        ...,
+        description="Numbers of the streams to try reading from when running a check operation.",
+        title="Stream Count",
+    )
+
+
 class ConcurrencyLevel(BaseModel):
     type: Optional[Literal["ConcurrencyLevel"]] = None
     default_concurrency: Union[int, str] = Field(
@@ -481,12 +490,24 @@ class RefreshTokenUpdater(BaseModel):
 
 class OAuthAuthenticator(BaseModel):
     type: Literal["OAuthAuthenticator"]
+    client_id_name: Optional[str] = Field(
+        "client_id",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_app_id"],
+        title="Client ID Property Name",
+    )
     client_id: str = Field(
         ...,
         description="The OAuth client ID. Fill it in the user inputs.",
         examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"],
         title="Client ID",
     )
+    client_secret_name: Optional[str] = Field(
+        "client_secret",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_app_secret"],
+        title="Client Secret Property Name",
+    )
     client_secret: str = Field(
         ...,
         description="The OAuth client secret. Fill it in the user inputs.",
@@ -496,6 +517,12 @@ class OAuthAuthenticator(BaseModel):
         ],
         title="Client Secret",
     )
+    refresh_token_name: Optional[str] = Field(
+        "refresh_token",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_app_refresh_value"],
+        title="Refresh Token Property Name",
+    )
     refresh_token: Optional[str] = Field(
         None,
         description="Credential artifact used to get a new access token.",
@@ -529,6 +556,12 @@ class OAuthAuthenticator(BaseModel):
         examples=["expires_in"],
         title="Token Expiry Property Name",
     )
+    grant_type_name: Optional[str] = Field(
+        "grant_type",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_grant_type"],
+        title="Grant Type Property Name",
+    )
     grant_type: Optional[str] = Field(
         "refresh_token",
         description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
@@ -547,6 +580,17 @@ class OAuthAuthenticator(BaseModel):
         ],
         title="Refresh Request Body",
     )
+    refresh_request_headers: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Headers of the request sent to get a new access token.",
+        examples=[
+            {
+                "Authorization": "<AUTH_TOKEN>",
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+        ],
+        title="Refresh Request Headers",
+    )
     scopes: Optional[List[str]] = Field(
         None,
         description="List of scopes that should be granted to the access token.",
@@ -675,6 +719,7 @@ class HttpResponseFilter(BaseModel):
 class TypesMap(BaseModel):
     target_type: Union[str, List[str]]
     current_type: Union[str, List[str]]
+    condition: Optional[str]
 
 
 class SchemaTypeIdentifier(BaseModel):
@@ -737,33 +782,43 @@ class KeysToSnakeCase(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class FlattenFields(BaseModel):
+    type: Literal["FlattenFields"]
+    flatten_lists: Optional[bool] = Field(
+        True,
+        description="Whether to flatten lists or leave it as is. Default is True.",
+        title="Flatten Lists",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class KeysReplace(BaseModel):
     type: Literal["KeysReplace"]
     old: str = Field(
         ...,
         description="Old value to replace.",
-        examples=[" ", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
+        examples=[
+            " ",
+            "{{ record.id }}",
+            "{{ config['id'] }}",
+            "{{ stream_slice['id'] }}",
+        ],
         title="Old value",
     )
     new: str = Field(
         ...,
         description="New value to set.",
-        examples=["_", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
+        examples=[
+            "_",
+            "{{ record.id }}",
+            "{{ config['id'] }}",
+            "{{ stream_slice['id'] }}",
+        ],
         title="New value",
     )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class FlattenFields(BaseModel):
-    type: Literal["FlattenFields"]
-    flatten_lists: Optional[bool] = Field(
-        True,
-        description="Whether to flatten lists or leave it as is. Default is True.",
-        title="Flatten Lists",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class IterableDecoder(BaseModel):
     type: Literal["IterableDecoder"]
 
@@ -849,8 +904,8 @@ class OauthConnectorInputSpecification(BaseModel):
         ...,
         description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
-            "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}",
+            "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",
+            "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}",
         ],
         title="Consent URL",
     )
@@ -864,14 +919,18 @@ class OauthConnectorInputSpecification(BaseModel):
         ...,
         description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
+            "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}"
         ],
         title="Access Token URL",
     )
     access_token_headers: Optional[Dict[str, Any]] = Field(
         None,
         description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
-        examples=[{"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"}],
+        examples=[
+            {
+                "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}"
+            }
+        ],
         title="Access Token Headers",
     )
     access_token_params: Optional[Dict[str, Any]] = Field(
@@ -879,15 +938,15 @@ class OauthConnectorInputSpecification(BaseModel):
         description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
         examples=[
             {
-                "{auth_code_key}": "{{auth_code_key}}",
-                "{client_id_key}": "{{client_id_key}}",
-                "{client_secret_key}": "{{client_secret_key}}",
+                "{{ auth_code_key }}": "{{ auth_code_value }}",
+                "{{ client_id_key }}": "{{ client_id_value }}",
+                "{{ client_secret_key }}": "{{ client_secret_value }}",
             }
         ],
         title="Access Token Query Params (Json Encoded)",
     )
-    extract_output: List[str] = Field(
-        ...,
+    extract_output: Optional[List[str]] = Field(
+        None,
         description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.",
         examples=[["access_token", "refresh_token", "other_field"]],
         title="Extract Output",
@@ -956,7 +1015,7 @@ class OAuthConfigSpecification(BaseModel):
     )
     oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
         None,
-        description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n  + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n  + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n  + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{auth_code_key}": "{{auth_code_key}}",\n              "{client_id_key}": "{{client_id_key}}",\n              "{client_secret_key}": "{{client_secret_key}}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
+        description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n  + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n  + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n  + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{{ auth_code_key }}": "{{ auth_code_value }}",\n              "{{ client_id_key }}": "{{ client_id_value }}",\n              "{{ client_secret_key }}": "{{ client_secret_value }}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
         title="DeclarativeOAuth Connector Specification",
     )
     complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
@@ -1163,6 +1222,11 @@ class LegacySessionTokenAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class JsonParser(BaseModel):
+    type: Literal["JsonParser"]
+    encoding: Optional[str] = "utf-8"
+
+
 class JsonLineParser(BaseModel):
     type: Literal["JsonLineParser"]
     encoding: Optional[str] = "utf-8"
@@ -1561,7 +1625,7 @@ class RecordSelector(BaseModel):
 
 class GzipParser(BaseModel):
     type: Literal["GzipParser"]
-    inner_parser: Union[JsonLineParser, CsvParser]
+    inner_parser: Union[JsonLineParser, CsvParser, JsonParser]
 
 
 class Spec(BaseModel):
@@ -1594,9 +1658,21 @@ class CompositeErrorHandler(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class ZipfileDecoder(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["ZipfileDecoder"]
+    parser: Union[GzipParser, JsonParser, JsonLineParser, CsvParser] = Field(
+        ...,
+        description="Parser to parse the decompressed data from the zipfile(s).",
+        title="Parser",
+    )
+
+
 class CompositeRawDecoder(BaseModel):
     type: Literal["CompositeRawDecoder"]
-    parser: Union[GzipParser, JsonLineParser, CsvParser]
+    parser: Union[GzipParser, JsonParser, JsonLineParser, CsvParser]
 
 
 class DeclarativeSource1(BaseModel):
@@ -1604,7 +1680,7 @@ class DeclarativeSource1(BaseModel):
         extra = Extra.forbid
 
     type: Literal["DeclarativeSource"]
-    check: CheckStream
+    check: Union[CheckStream, CheckDynamicStream]
     streams: List[DeclarativeStream]
     dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None
     version: str = Field(
@@ -1630,7 +1706,7 @@ class DeclarativeSource2(BaseModel):
         extra = Extra.forbid
 
     type: Literal["DeclarativeSource"]
-    check: CheckStream
+    check: Union[CheckStream, CheckDynamicStream]
     streams: Optional[List[DeclarativeStream]] = None
     dynamic_streams: List[DynamicDeclarativeStream]
     version: str = Field(
@@ -1799,7 +1875,7 @@ class SessionTokenAuthenticator(BaseModel):
         description="Authentication method to use for requests sent to the API, specifying how to inject the session token.",
         title="Data Request Authentication",
     )
-    decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field(
+    decoder: Optional[Union[JsonDecoder, XmlDecoder, CompositeRawDecoder]] = Field(
         None, description="Component used to decode the response.", title="Decoder"
     )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
@@ -2004,6 +2080,7 @@ class SimpleRetriever(BaseModel):
             XmlDecoder,
             GzipJsonDecoder,
             CompositeRawDecoder,
+            ZipfileDecoder,
         ]
     ] = Field(
         None,
@@ -2040,6 +2117,10 @@ class AsyncRetriever(BaseModel):
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.",
     )
+    url_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
+        None,
+        description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.",
+    )
     download_requester: Union[CustomRequester, HttpRequester] = Field(
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.",
@@ -2076,6 +2157,8 @@ class AsyncRetriever(BaseModel):
             IterableDecoder,
             XmlDecoder,
             GzipJsonDecoder,
+            CompositeRawDecoder,
+            ZipfileDecoder,
         ]
     ] = Field(
         None,
@@ -2090,6 +2173,8 @@ class AsyncRetriever(BaseModel):
             IterableDecoder,
             XmlDecoder,
             GzipJsonDecoder,
+            CompositeRawDecoder,
+            ZipfileDecoder,
         ]
     ] = Field(
         None,