airbyte-cdk 6.18.0.dev1__tar.gz → 6.18.1__tar.gz

{airbyte_cdk-6.18.0.dev1 → airbyte_cdk-6.18.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-cdk
-Version: 6.18.0.dev1
+Version: 6.18.1
 Summary: A framework for writing Airbyte Connectors.
 License: MIT
 Keywords: airbyte,connector-development-kit,cdk

{airbyte_cdk-6.18.0.dev1 → airbyte_cdk-6.18.1}/airbyte_cdk/sources/declarative/auth/oauth.py

@@ -56,8 +56,12 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
     token_expiry_is_time_of_expiration: bool = False
     access_token_name: Union[InterpolatedString, str] = "access_token"
     access_token_value: Optional[Union[InterpolatedString, str]] = None
+    client_id_name: Union[InterpolatedString, str] = "client_id"
+    client_secret_name: Union[InterpolatedString, str] = "client_secret"
     expires_in_name: Union[InterpolatedString, str] = "expires_in"
+    refresh_token_name: Union[InterpolatedString, str] = "refresh_token"
     refresh_request_body: Optional[Mapping[str, Any]] = None
+    grant_type_name: Union[InterpolatedString, str] = "grant_type"
     grant_type: Union[InterpolatedString, str] = "refresh_token"
     message_repository: MessageRepository = NoopMessageRepository()
 
@@ -69,8 +73,15 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
             )
         else:
             self._token_refresh_endpoint = None
+        self._client_id_name = InterpolatedString.create(self.client_id_name, parameters=parameters)
         self._client_id = InterpolatedString.create(self.client_id, parameters=parameters)
+        self._client_secret_name = InterpolatedString.create(
+            self.client_secret_name, parameters=parameters
+        )
         self._client_secret = InterpolatedString.create(self.client_secret, parameters=parameters)
+        self._refresh_token_name = InterpolatedString.create(
+            self.refresh_token_name, parameters=parameters
+        )
         if self.refresh_token is not None:
             self._refresh_token: Optional[InterpolatedString] = InterpolatedString.create(
                 self.refresh_token, parameters=parameters
@@ -83,6 +94,9 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
         self.expires_in_name = InterpolatedString.create(
             self.expires_in_name, parameters=parameters
         )
+        self.grant_type_name = InterpolatedString.create(
+            self.grant_type_name, parameters=parameters
+        )
         self.grant_type = InterpolatedString.create(self.grant_type, parameters=parameters)
         self._refresh_request_body = InterpolatedMapping(
             self.refresh_request_body or {}, parameters=parameters
@@ -122,18 +136,27 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
             return refresh_token_endpoint
         return None
 
+    def get_client_id_name(self) -> str:
+        return self._client_id_name.eval(self.config)  # type: ignore # eval returns a string in this context
+
     def get_client_id(self) -> str:
         client_id: str = self._client_id.eval(self.config)
         if not client_id:
             raise ValueError("OAuthAuthenticator was unable to evaluate client_id parameter")
         return client_id
 
+    def get_client_secret_name(self) -> str:
+        return self._client_secret_name.eval(self.config)  # type: ignore # eval returns a string in this context
+
     def get_client_secret(self) -> str:
         client_secret: str = self._client_secret.eval(self.config)
         if not client_secret:
             raise ValueError("OAuthAuthenticator was unable to evaluate client_secret parameter")
         return client_secret
 
+    def get_refresh_token_name(self) -> str:
+        return self._refresh_token_name.eval(self.config)  # type: ignore # eval returns a string in this context
+
     def get_refresh_token(self) -> Optional[str]:
         return None if self._refresh_token is None else str(self._refresh_token.eval(self.config))
 
@@ -146,6 +169,9 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
     def get_expires_in_name(self) -> str:
         return self.expires_in_name.eval(self.config)  # type: ignore # eval returns a string in this context
 
+    def get_grant_type_name(self) -> str:
+        return self.grant_type_name.eval(self.config)  # type: ignore # eval returns a string in this context
+
     def get_grant_type(self) -> str:
         return self.grant_type.eval(self.config)  # type: ignore # eval returns a string in this context
 

{airbyte_cdk-6.18.0.dev1 → airbyte_cdk-6.18.1}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -678,7 +678,7 @@ definitions:
     properties:
       type:
         type: string
-        enum: [CustomSchemaNormalization]
+        enum: [ CustomSchemaNormalization ]
       class_name:
         title: Class Name
         description: Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.
@@ -1047,6 +1047,13 @@ definitions:
       type:
         type: string
         enum: [OAuthAuthenticator]
+      client_id_name:
+        title: Client ID Property Name
+        description: The name of the property to use to refresh the `access_token`.
+        type: string
+        default: "client_id"
+        examples:
+          - custom_app_id
       client_id:
         title: Client ID
         description: The OAuth client ID. Fill it in the user inputs.
@@ -1054,6 +1061,13 @@ definitions:
         examples:
           - "{{ config['client_id }}"
           - "{{ config['credentials']['client_id }}"
+      client_secret_name:
+        title: Client Secret Property Name
+        description: The name of the property to use to refresh the `access_token`.
+        type: string
+        default: "client_secret"
+        examples:
+          - custom_app_secret
       client_secret:
         title: Client Secret
         description: The OAuth client secret. Fill it in the user inputs.
@@ -1061,6 +1075,13 @@ definitions:
         examples:
           - "{{ config['client_secret }}"
           - "{{ config['credentials']['client_secret }}"
+      refresh_token_name:
+        title: Refresh Token Property Name
+        description: The name of the property to use to refresh the `access_token`.
+        type: string
+        default: "refresh_token"
+        examples:
+          - custom_app_refresh_value
       refresh_token:
         title: Refresh Token
         description: Credential artifact used to get a new access token.
@@ -1094,6 +1115,13 @@ definitions:
         default: "expires_in"
         examples:
           - expires_in
+      grant_type_name:
+        title: Grant Type Property Name
+        description: The name of the property to use to refresh the `access_token`.
+        type: string
+        default: "grant_type"
+        examples:
+          - custom_grant_type
       grant_type:
         title: Grant Type
         description: Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.
@@ -2014,20 +2042,6 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
-  JsonParser:
-    title: JsonParser
-    description: Parser used for parsing str, bytes, or bytearray data and returning data in a dictionary format.
-    type: object
-    additionalProperties: true
-    required:
-      - type
-    properties:
-      type:
-        type: string
-        enum: [JsonParser]
-      encoding:
-        type: string
-        default: utf-8
   ListPartitionRouter:
     title: List Partition Router
     description: A Partition router that specifies a list of attributes where each attribute describes a portion of the complete data set for a stream. During a sync, each value is iterated over and can be used as input to outbound API requests.
@@ -2218,15 +2232,15 @@ definitions:
           Pertains to the fields defined by the connector relating to the OAuth flow.
 
           Interpolation capabilities:
-          - The variables placeholders are declared as `{my_var}`.
-          - The nested resolution variables like `{{my_nested_var}}` is allowed as well.
+          - The variables placeholders are declared as `{{my_var}}`.
+          - The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.
 
           - The allowed interpolation context is:
-            + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}
-            + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}
-            + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}
-            + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}
-            + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}
+            + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}
+            + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}
+            + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}
+            + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}
+            + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}
 
           Examples:
             - The TikTok Marketing DeclarativeOAuth spec:
@@ -2235,12 +2249,12 @@ definitions:
                 "type": "object",
                 "additionalProperties": false,
                 "properties": {
-                    "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
+                    "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",
                     "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",
                     "access_token_params": {
-                        "{auth_code_key}": "{{auth_code_key}}",
-                        "{client_id_key}": "{{client_id_key}}",
-                        "{client_secret_key}": "{{client_secret_key}}"
+                        "{{ auth_code_key }}": "{{ auth_code_value }}",
+                        "{{ client_id_key }}": "{{ client_id_value }}",
+                        "{{ client_secret_key }}": "{{ client_secret_value }}"
                     },
                     "access_token_headers": {
                         "Content-Type": "application/json",
@@ -2258,7 +2272,6 @@ definitions:
         required:
           - consent_url
           - access_token_url
-          - extract_output
         properties:
           consent_url:
             title: Consent URL
@@ -2267,8 +2280,8 @@ definitions:
               The DeclarativeOAuth Specific string URL string template to initiate the authentication.
               The placeholders are replaced during the processing to provide neccessary values.
             examples:
-              - https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}
-              - https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}
+              - https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}
+              - https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}
           scope:
             title: Scopes
             type: string
@@ -2283,7 +2296,7 @@ definitions:
               The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.
               The placeholders are replaced during the processing to provide neccessary values.
             examples:
-              - https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}
+              - https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}
           access_token_headers:
             title: Access Token Headers
             type: object
@@ -2292,7 +2305,7 @@ definitions:
               The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
             examples:
               - {
-                  "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}",
+                  "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}",
                 }
           access_token_params:
             title: Access Token Query Params (Json Encoded)
@@ -2303,9 +2316,9 @@ definitions:
               When this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.
             examples:
               - {
-                  "{auth_code_key}": "{{auth_code_key}}",
-                  "{client_id_key}": "{{client_id_key}}",
-                  "{client_secret_key}": "{{client_secret_key}}",
+                  "{{ auth_code_key }}": "{{ auth_code_value }}",
+                  "{{ client_id_key }}": "{{ client_id_value }}",
+                  "{{ client_secret_key }}": "{{ client_secret_value }}",
                 }
           extract_output:
             title: Extract Output
@@ -2991,6 +3004,11 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/CustomRequester"
           - "$ref": "#/definitions/HttpRequester"
+      url_requester:
+        description: Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.
+        anyOf:
+          - "$ref": "#/definitions/CustomRequester"
+          - "$ref": "#/definitions/HttpRequester"
       download_requester:
         description: Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.
         anyOf:

{airbyte_cdk-6.18.0.dev1 → airbyte_cdk-6.18.1}/airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -7,12 +7,9 @@ from dataclasses import dataclass
 from io import BufferedIOBase, TextIOWrapper
 from typing import Any, Generator, MutableMapping, Optional
 
-import orjson
 import requests
 
-from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
-from airbyte_cdk.utils import AirbyteTracedException
 
 logger = logging.getLogger("airbyte")
 
@@ -45,31 +42,6 @@ class GzipParser(Parser):
             yield from self.inner_parser.parse(gzipobj)
 
 
-@dataclass
-class JsonParser(Parser):
-    encoding: str = "utf-8"
-
-    def parse(self, data: BufferedIOBase) -> Generator[MutableMapping[str, Any], None, None]:
-        raw_data = data.read()
-        try:
-            body_json = orjson.loads(raw_data.decode(self.encoding))
-        except Exception:
-            try:
-                body_json = json.loads(raw_data.decode(self.encoding))
-            except Exception as exc:
-                raise AirbyteTracedException(
-                    message="Response JSON data failed to be parsed. See logs for more inforation.",
-                    internal_message=f"Response JSON data faild to be parsed: {exc=}, {raw_data=}",
-                    failure_type=FailureType.system_error,
-                    exception=exc,
-                )
-
-        if isinstance(body_json, list):
-            yield from body_json
-        else:
-            yield from [body_json]
-
-
 @dataclass
 class JsonLineParser(Parser):
     encoding: Optional[str] = "utf-8"

{airbyte_cdk-6.18.0.dev1 → airbyte_cdk-6.18.1}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -481,12 +481,24 @@ class RefreshTokenUpdater(BaseModel):
 
 class OAuthAuthenticator(BaseModel):
     type: Literal["OAuthAuthenticator"]
+    client_id_name: Optional[str] = Field(
+        "client_id",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_app_id"],
+        title="Client ID Property Name",
+    )
     client_id: str = Field(
         ...,
         description="The OAuth client ID. Fill it in the user inputs.",
         examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"],
         title="Client ID",
     )
+    client_secret_name: Optional[str] = Field(
+        "client_secret",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_app_secret"],
+        title="Client Secret Property Name",
+    )
     client_secret: str = Field(
         ...,
         description="The OAuth client secret. Fill it in the user inputs.",
@@ -496,6 +508,12 @@ class OAuthAuthenticator(BaseModel):
         ],
         title="Client Secret",
     )
+    refresh_token_name: Optional[str] = Field(
+        "refresh_token",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_app_refresh_value"],
+        title="Refresh Token Property Name",
+    )
     refresh_token: Optional[str] = Field(
         None,
         description="Credential artifact used to get a new access token.",
@@ -529,6 +547,12 @@ class OAuthAuthenticator(BaseModel):
         examples=["expires_in"],
         title="Token Expiry Property Name",
     )
+    grant_type_name: Optional[str] = Field(
+        "grant_type",
+        description="The name of the property to use to refresh the `access_token`.",
+        examples=["custom_grant_type"],
+        title="Grant Type Property Name",
+    )
     grant_type: Optional[str] = Field(
         "refresh_token",
         description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
@@ -805,14 +829,6 @@ class GzipJsonDecoder(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class JsonParser(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["JsonParser"]
-    encoding: Optional[str] = "utf-8"
-
-
 class MinMaxDatetime(BaseModel):
     type: Literal["MinMaxDatetime"]
     datetime: str = Field(
@@ -867,8 +883,8 @@ class OauthConnectorInputSpecification(BaseModel):
         ...,
         description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
-            "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}",
+            "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",
+            "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}",
         ],
         title="Consent URL",
     )
@@ -882,14 +898,18 @@ class OauthConnectorInputSpecification(BaseModel):
         ...,
         description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
+            "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}"
         ],
         title="Access Token URL",
     )
     access_token_headers: Optional[Dict[str, Any]] = Field(
         None,
         description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
-        examples=[{"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"}],
+        examples=[
+            {
+                "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}"
+            }
+        ],
         title="Access Token Headers",
     )
     access_token_params: Optional[Dict[str, Any]] = Field(
@@ -897,15 +917,15 @@ class OauthConnectorInputSpecification(BaseModel):
         description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
         examples=[
             {
-                "{auth_code_key}": "{{auth_code_key}}",
-                "{client_id_key}": "{{client_id_key}}",
-                "{client_secret_key}": "{{client_secret_key}}",
+                "{{ auth_code_key }}": "{{ auth_code_value }}",
+                "{{ client_id_key }}": "{{ client_id_value }}",
+                "{{ client_secret_key }}": "{{ client_secret_value }}",
             }
         ],
         title="Access Token Query Params (Json Encoded)",
     )
-    extract_output: List[str] = Field(
-        ...,
+    extract_output: Optional[List[str]] = Field(
+        None,
         description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.",
         examples=[["access_token", "refresh_token", "other_field"]],
         title="Extract Output",
@@ -974,7 +994,7 @@ class OAuthConfigSpecification(BaseModel):
     )
     oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
         None,
-        description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n  + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n  + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n  + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{auth_code_key}": "{{auth_code_key}}",\n              "{client_id_key}": "{{client_id_key}}",\n              "{client_secret_key}": "{{client_secret_key}}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
+        description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n  + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n  + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n  + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{{ auth_code_key }}": "{{ auth_code_value }}",\n              "{{ client_id_key }}": "{{ client_id_value }}",\n              "{{ client_secret_key }}": "{{ client_secret_value }}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
         title="DeclarativeOAuth Connector Specification",
     )
     complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
@@ -2058,6 +2078,10 @@ class AsyncRetriever(BaseModel):
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.",
     )
+    url_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
+        None,
+        description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.",
+    )
     download_requester: Union[CustomRequester, HttpRequester] = Field(
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.",

{airbyte_cdk-6.18.0.dev1 → airbyte_cdk-6.18.1}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -72,7 +72,6 @@ from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import (
     CsvParser,
     GzipParser,
     JsonLineParser,
-    JsonParser,
 )
 from airbyte_cdk.sources.declarative.extractors import (
     DpathExtractor,
@@ -248,9 +247,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     JsonLineParser as JsonLineParserModel,
 )
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
-    JsonParser as JsonParserModel,
-)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     JwtAuthenticator as JwtAuthenticatorModel,
 )
@@ -526,7 +522,6 @@ class ModelToComponentFactory:
             JsonDecoderModel: self.create_json_decoder,
             JsonlDecoderModel: self.create_jsonl_decoder,
             JsonLineParserModel: self.create_json_line_parser,
-            JsonParserModel: self.create_json_parser,
             GzipJsonDecoderModel: self.create_gzipjson_decoder,
             GzipParserModel: self.create_gzip_parser,
             KeysToLowerModel: self.create_keys_to_lower_transformation,
@@ -1753,11 +1748,6 @@ class ModelToComponentFactory:
     def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> JsonDecoder:
         return JsonDecoder(parameters={})
 
-    @staticmethod
-    def create_json_parser(model: JsonParserModel, config: Config, **kwargs: Any) -> JsonParser:
-        encoding = model.encoding or "utf-8"
-        return JsonParser(encoding=encoding)
-
     @staticmethod
     def create_jsonl_decoder(
         model: JsonlDecoderModel, config: Config, **kwargs: Any
@@ -1895,15 +1885,24 @@ class ModelToComponentFactory:
                 expires_in_name=InterpolatedString.create(
                     model.expires_in_name or "expires_in", parameters=model.parameters or {}
                 ).eval(config),
+                client_id_name=InterpolatedString.create(
+                    model.client_id_name or "client_id", parameters=model.parameters or {}
+                ).eval(config),
                 client_id=InterpolatedString.create(
                     model.client_id, parameters=model.parameters or {}
                 ).eval(config),
+                client_secret_name=InterpolatedString.create(
+                    model.client_secret_name or "client_secret", parameters=model.parameters or {}
+                ).eval(config),
                 client_secret=InterpolatedString.create(
                     model.client_secret, parameters=model.parameters or {}
                 ).eval(config),
                 access_token_config_path=model.refresh_token_updater.access_token_config_path,
                 refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path,
                 token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path,
+                grant_type_name=InterpolatedString.create(
+                    model.grant_type_name or "grant_type", parameters=model.parameters or {}
+                ).eval(config),
                 grant_type=InterpolatedString.create(
                     model.grant_type or "refresh_token", parameters=model.parameters or {}
                 ).eval(config),
@@ -1921,11 +1920,15 @@ class ModelToComponentFactory:
         return DeclarativeOauth2Authenticator(  # type: ignore
             access_token_name=model.access_token_name or "access_token",
             access_token_value=model.access_token_value,
+            client_id_name=model.client_id_name or "client_id",
             client_id=model.client_id,
+            client_secret_name=model.client_secret_name or "client_secret",
             client_secret=model.client_secret,
             expires_in_name=model.expires_in_name or "expires_in",
+            grant_type_name=model.grant_type_name or "grant_type",
             grant_type=model.grant_type or "refresh_token",
             refresh_request_body=model.refresh_request_body,
+            refresh_token_name=model.refresh_token_name or "refresh_token",
             refresh_token=model.refresh_token,
             scopes=model.scopes,
             token_expiry_date=model.token_expiry_date,
@@ -2297,7 +2300,7 @@ class ModelToComponentFactory:
                 extractor=download_extractor,
                 name=name,
                 record_filter=None,
-                transformations=[],
+                transformations=transformations,
                 schema_normalization=TypeTransformer(TransformConfig.NoTransform),
                 config=config,
                 parameters={},
@@ -2334,6 +2337,16 @@ class ModelToComponentFactory:
             if model.delete_requester
             else None
         )
+        url_requester = (
+            self._create_component_from_model(
+                model=model.url_requester,
+                decoder=decoder,
+                config=config,
+                name=f"job extract_url - {name}",
+            )
+            if model.url_requester
+            else None
+        )
         status_extractor = self._create_component_from_model(
             model=model.status_extractor, decoder=decoder, config=config, name=name
         )
@@ -2344,6 +2357,7 @@ class ModelToComponentFactory:
             creation_requester=creation_requester,
             polling_requester=polling_requester,
             download_retriever=download_retriever,
+            url_requester=url_requester,
             abort_requester=abort_requester,
             delete_requester=delete_requester,
             status_extractor=status_extractor,

airbyte_cdk-6.18.1/airbyte_cdk/sources/declarative/requesters/README.md

@@ -0,0 +1,57 @@
+# AsyncHttpJobRepository sequence diagram
+
+- Components marked as optional are not required and can be ignored.
+- if `url_requester` is not provided, `urls_extractor` will get urls from the `polling_job_response`
+- interpolation_context, e.g. `create_job_response` or `polling_job_response` can be obtained from stream_slice
+ 
+
+```mermaid
+---
+title: AsyncHttpJobRepository Sequence Diagram
+---
+sequenceDiagram 
+    participant AsyncHttpJobRepository as AsyncOrchestrator
+    participant CreationRequester as creation_requester
+    participant PollingRequester as polling_requester
+    participant UrlRequester as url_requester (Optional)
+    participant DownloadRetriever as download_retriever
+    participant AbortRequester as abort_requester (Optional)
+    participant DeleteRequester as delete_requester (Optional)
+    participant Reporting Server as Async Reporting Server
+
+    AsyncHttpJobRepository ->> CreationRequester: Initiate job creation
+    CreationRequester ->> Reporting Server: Create job request
+    Reporting Server -->> CreationRequester: Job ID response
+    CreationRequester -->> AsyncHttpJobRepository: Job ID
+
+    loop Poll for job status
+        AsyncHttpJobRepository ->> PollingRequester: Check job status
+        PollingRequester ->> Reporting Server: Status request (interpolation_context: `create_job_response`)
+        Reporting Server -->> PollingRequester: Status response
+        PollingRequester -->> AsyncHttpJobRepository: Job status
+    end
+
+    alt Status: Ready
+        AsyncHttpJobRepository ->> UrlRequester: Request download URLs (if applicable)
+        UrlRequester ->> Reporting Server: URL request (interpolation_context: `polling_job_response`)
+        Reporting Server -->> UrlRequester: Download URLs
+        UrlRequester -->> AsyncHttpJobRepository: Download URLs
+
+        AsyncHttpJobRepository ->> DownloadRetriever: Download reports
+        DownloadRetriever ->> Reporting Server: Retrieve report data (interpolation_context: `url`)
+        Reporting Server -->> DownloadRetriever: Report data
+        DownloadRetriever -->> AsyncHttpJobRepository: Report data
+    else Status: Failed
+        AsyncHttpJobRepository ->> AbortRequester: Send abort request
+        AbortRequester ->> Reporting Server: Abort job
+        Reporting Server -->> AbortRequester: Abort confirmation
+        AbortRequester -->> AsyncHttpJobRepository: Confirmation
+    end
+
+    AsyncHttpJobRepository ->> DeleteRequester: Send delete job request
+    DeleteRequester ->> Reporting Server: Delete job
+    Reporting Server -->> DeleteRequester: Deletion confirmation
+    DeleteRequester -->> AsyncHttpJobRepository: Confirmation
+
+
+```