airbyte-cdk 6.20.0__py3-none-any.whl → 6.20.2.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -0,0 +1,331 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import copy
+import logging
+import threading
+from collections import OrderedDict
+from copy import deepcopy
+from datetime import timedelta
+from typing import Any, Callable, Iterable, Mapping, MutableMapping, Optional
+
+from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
+from airbyte_cdk.sources.declarative.incremental.global_substream_cursor import (
+    Timer,
+    iterate_with_last_flag_and_state,
+)
+from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
+from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.streams.checkpoint.per_partition_key_serializer import (
+    PerPartitionKeySerializer,
+)
+from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, Cursor, CursorField
+from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
+from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
+
+logger = logging.getLogger("airbyte")
+
+
+class ConcurrentCursorFactory:
+    def __init__(self, create_function: Callable[..., ConcurrentCursor]):
+        self._create_function = create_function
+
+    def create(
+        self, stream_state: Mapping[str, Any], runtime_lookback_window: Any
+    ) -> ConcurrentCursor:
+        return self._create_function(
+            stream_state=stream_state, runtime_lookback_window=runtime_lookback_window
+        )
+
+
+class ConcurrentPerPartitionCursor(Cursor):
+    """
+    Manages state per partition when a stream has many partitions, preventing data loss or duplication.
+
+    Attributes:
+        DEFAULT_MAX_PARTITIONS_NUMBER (int): Maximum number of partitions to retain in memory (default is 10,000).
+
+    - **Partition Limitation Logic**
+      Ensures the number of tracked partitions does not exceed the specified limit to prevent memory overuse. Oldest partitions are removed when the limit is reached.
+
+    - **Global Cursor Fallback**
+      New partitions use global state as the initial state to progress the state for deleted or new partitions. The history data added after the initial sync will be missing.
+    """
+
+    DEFAULT_MAX_PARTITIONS_NUMBER = 10000
+    _NO_STATE: Mapping[str, Any] = {}
+    _NO_CURSOR_STATE: Mapping[str, Any] = {}
+    _GLOBAL_STATE_KEY = "state"
+    _PERPARTITION_STATE_KEY = "states"
+    _KEY = 0
+    _VALUE = 1
+
+    def __init__(
+        self,
+        cursor_factory: ConcurrentCursorFactory,
+        partition_router: PartitionRouter,
+        stream_name: str,
+        stream_namespace: Optional[str],
+        stream_state: Any,
+        message_repository: MessageRepository,
+        connector_state_manager: ConnectorStateManager,
+        cursor_field: CursorField,
+    ) -> None:
+        self._global_cursor: Optional[StreamState] = {}
+        self._stream_name = stream_name
+        self._stream_namespace = stream_namespace
+        self._message_repository = message_repository
+        self._connector_state_manager = connector_state_manager
+        self._cursor_field = cursor_field
+
+        self._cursor_factory = cursor_factory
+        self._partition_router = partition_router
+
+        # The dict is ordered to ensure that once the maximum number of partitions is reached,
+        # the oldest partitions can be efficiently removed, maintaining the most recent partitions.
+        self._cursor_per_partition: OrderedDict[str, ConcurrentCursor] = OrderedDict()
+        self._semaphore_per_partition: OrderedDict[str, threading.Semaphore] = OrderedDict()
+        self._finished_partitions: set[str] = set()
+        self._lock = threading.Lock()
+        self._timer = Timer()
+        self._new_global_cursor: Optional[StreamState] = None
+        self._lookback_window: int = 0
+        self._parent_state: Optional[StreamState] = None
+        self._over_limit: int = 0
+        self._partition_serializer = PerPartitionKeySerializer()
+
+        self._set_initial_state(stream_state)
+
+    @property
+    def cursor_field(self) -> CursorField:
+        return self._cursor_field
+
+    @property
+    def state(self) -> MutableMapping[str, Any]:
+        states = []
+        for partition_tuple, cursor in self._cursor_per_partition.items():
+            if cursor.state:
+                states.append(
+                    {
+                        "partition": self._to_dict(partition_tuple),
+                        "cursor": copy.deepcopy(cursor.state),
+                    }
+                )
+        state: dict[str, Any] = {self._PERPARTITION_STATE_KEY: states}
+
+        if self._global_cursor:
+            state[self._GLOBAL_STATE_KEY] = self._global_cursor
+        if self._lookback_window is not None:
+            state["lookback_window"] = self._lookback_window
+        if self._parent_state is not None:
+            state["parent_state"] = self._parent_state
+        return state
+
+    def close_partition(self, partition: Partition) -> None:
+        # Attempt to retrieve the stream slice
+        stream_slice: Optional[StreamSlice] = partition.to_slice()  # type: ignore[assignment]
+
+        # Ensure stream_slice is not None
+        if stream_slice is None:
+            raise ValueError("stream_slice cannot be None")
+
+        partition_key = self._to_partition_key(stream_slice.partition)
+        self._cursor_per_partition[partition_key].close_partition(partition=partition)
+        with self._lock:
+            self._semaphore_per_partition[partition_key].acquire()
+            cursor = self._cursor_per_partition[partition_key]
+            if (
+                partition_key in self._finished_partitions
+                and self._semaphore_per_partition[partition_key]._value == 0
+            ):
+                if (
+                    self._new_global_cursor is None
+                    or self._new_global_cursor[self.cursor_field.cursor_field_key]
+                    < cursor.state[self.cursor_field.cursor_field_key]
+                ):
+                    self._new_global_cursor = copy.deepcopy(cursor.state)
+
+    def ensure_at_least_one_state_emitted(self) -> None:
+        """
+        The platform expect to have at least one state message on successful syncs. Hence, whatever happens, we expect this method to be
+        called.
+        """
+        if not any(
+            semaphore_item[1]._value for semaphore_item in self._semaphore_per_partition.items()
+        ):
+            self._global_cursor = self._new_global_cursor
+            self._lookback_window = self._timer.finish()
+            self._parent_state = self._partition_router.get_stream_state()
+        self._emit_state_message()
+
+    def _emit_state_message(self) -> None:
+        self._connector_state_manager.update_state_for_stream(
+            self._stream_name,
+            self._stream_namespace,
+            self.state,
+        )
+        state_message = self._connector_state_manager.create_state_message(
+            self._stream_name, self._stream_namespace
+        )
+        self._message_repository.emit_message(state_message)
+
+    def stream_slices(self) -> Iterable[StreamSlice]:
+        if self._timer.is_running():
+            raise RuntimeError("stream_slices has been executed more than once.")
+
+        slices = self._partition_router.stream_slices()
+        self._timer.start()
+        for partition in slices:
+            yield from self._generate_slices_from_partition(partition)
+
+    def _generate_slices_from_partition(self, partition: StreamSlice) -> Iterable[StreamSlice]:
+        # Ensure the maximum number of partitions is not exceeded
+        self._ensure_partition_limit()
+
+        cursor = self._cursor_per_partition.get(self._to_partition_key(partition.partition))
+        if not cursor:
+            cursor = self._create_cursor(
+                self._global_cursor,
+                self._lookback_window if self._global_cursor else self._NO_CURSOR_STATE,
+            )
+            self._cursor_per_partition[self._to_partition_key(partition.partition)] = cursor
+            self._semaphore_per_partition[self._to_partition_key(partition.partition)] = (
+                threading.Semaphore(0)
+            )
+
+        for cursor_slice, is_last_slice, _ in iterate_with_last_flag_and_state(
+            cursor.stream_slices(),
+            lambda: None,
+        ):
+            self._semaphore_per_partition[self._to_partition_key(partition.partition)].release()
+            if is_last_slice:
+                self._finished_partitions.add(self._to_partition_key(partition.partition))
+            yield StreamSlice(
+                partition=partition, cursor_slice=cursor_slice, extra_fields=partition.extra_fields
+            )
+
+    def _ensure_partition_limit(self) -> None:
+        """
+        Ensure the maximum number of partitions is not exceeded. If so, the oldest added partition will be dropped.
+        """
+        while len(self._cursor_per_partition) > self.DEFAULT_MAX_PARTITIONS_NUMBER - 1:
+            self._over_limit += 1
+            oldest_partition = self._cursor_per_partition.popitem(last=False)[
+                0
+            ]  # Remove the oldest partition
+            logger.warning(
+                f"The maximum number of partitions has been reached. Dropping the oldest partition: {oldest_partition}. Over limit: {self._over_limit}."
+            )
+
+    def limit_reached(self) -> bool:
+        return self._over_limit > self.DEFAULT_MAX_PARTITIONS_NUMBER
+
+    def _set_initial_state(self, stream_state: StreamState) -> None:
+        """
+        Initialize the cursor's state using the provided `stream_state`.
+
+        This method supports global and per-partition state initialization.
+
+        - **Global State**: If `states` is missing, the `state` is treated as global and applied to all partitions.
+          The `global state` holds a single cursor position representing the latest processed record across all partitions.
+
+        - **Lookback Window**: Configured via `lookback_window`, it defines the period (in seconds) for reprocessing records.
+          This ensures robustness in case of upstream data delays or reordering. If not specified, it defaults to 0.
+
+        - **Per-Partition State**: If `states` is present, each partition's cursor state is initialized separately.
+
+        - **Parent State**: (if available) Used to initialize partition routers based on parent streams.
+
+        Args:
+            stream_state (StreamState): The state of the streams to be set. The format of the stream state should be:
+                {
+                    "states": [
+                        {
+                            "partition": {
+                                "partition_key": "value"
+                            },
+                            "cursor": {
+                                "last_updated": "2023-05-27T00:00:00Z"
+                            }
+                        }
+                    ],
+                    "state": {
+                        "last_updated": "2023-05-27T00:00:00Z"
+                    },
+                    lookback_window: 10,
+                    "parent_state": {
+                        "parent_stream_name": {
+                            "last_updated": "2023-05-27T00:00:00Z"
+                        }
+                    }
+                }
+        """
+        if not stream_state:
+            return
+
+        if self._PERPARTITION_STATE_KEY not in stream_state:
+            # We assume that `stream_state` is in a global format that can be applied to all partitions.
+            # Example: {"global_state_format_key": "global_state_format_value"}
+            self._global_cursor = deepcopy(stream_state)
+            self._new_global_cursor = deepcopy(stream_state)
+
+        else:
+            self._lookback_window = int(stream_state.get("lookback_window", 0))
+
+            for state in stream_state[self._PERPARTITION_STATE_KEY]:
+                self._cursor_per_partition[self._to_partition_key(state["partition"])] = (
+                    self._create_cursor(state["cursor"])
+                )
+                self._semaphore_per_partition[self._to_partition_key(state["partition"])] = (
+                    threading.Semaphore(0)
+                )
+
+            # set default state for missing partitions if it is per partition with fallback to global
+            if self._GLOBAL_STATE_KEY in stream_state:
+                self._global_cursor = deepcopy(stream_state[self._GLOBAL_STATE_KEY])
+                self._new_global_cursor = deepcopy(stream_state[self._GLOBAL_STATE_KEY])
+
+        # Set parent state for partition routers based on parent streams
+        self._partition_router.set_initial_state(stream_state)
+
+    def observe(self, record: Record) -> None:
+        if not record.associated_slice:
+            raise ValueError(
+                "Invalid state as stream slices that are emitted should refer to an existing cursor"
+            )
+        self._cursor_per_partition[
+            self._to_partition_key(record.associated_slice.partition)
+        ].observe(record)
+
+    def _to_partition_key(self, partition: Mapping[str, Any]) -> str:
+        return self._partition_serializer.to_partition_key(partition)
+
+    def _to_dict(self, partition_key: str) -> Mapping[str, Any]:
+        return self._partition_serializer.to_partition(partition_key)
+
+    def _create_cursor(
+        self, cursor_state: Any, runtime_lookback_window: Any = None
+    ) -> ConcurrentCursor:
+        if runtime_lookback_window:
+            runtime_lookback_window = timedelta(seconds=runtime_lookback_window)
+        cursor = self._cursor_factory.create(
+            stream_state=deepcopy(cursor_state), runtime_lookback_window=runtime_lookback_window
+        )
+        return cursor
+
+    def should_be_synced(self, record: Record) -> bool:
+        return self._get_cursor(record).should_be_synced(record)
+
+    def _get_cursor(self, record: Record) -> ConcurrentCursor:
+        if not record.associated_slice:
+            raise ValueError(
+                "Invalid state as stream slices that are emitted should refer to an existing cursor"
+            )
+        partition_key = self._to_partition_key(record.associated_slice.partition)
+        if partition_key not in self._cursor_per_partition:
+            raise ValueError(
+                "Invalid state as stream slices that are emitted should refer to an existing cursor"
+            )
+        cursor = self._cursor_per_partition[partition_key]
+        return cursor

airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py

@@ -64,6 +64,9 @@ class Timer:
         else:
             raise RuntimeError("Global substream cursor timer not started")
 
+    def is_running(self) -> bool:
+        return self._start is not None
+
 
 class GlobalSubstreamCursor(DeclarativeCursor):
     """

airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py

@@ -303,6 +303,21 @@ class PerPartitionCursor(DeclarativeCursor):
             raise ValueError("A partition needs to be provided in order to get request body json")
 
     def should_be_synced(self, record: Record) -> bool:
+        if (
+            record.associated_slice
+            and self._to_partition_key(record.associated_slice.partition)
+            not in self._cursor_per_partition
+        ):
+            partition_state = (
+                self._state_to_migrate_from
+                if self._state_to_migrate_from
+                else self._NO_CURSOR_STATE
+            )
+            cursor = self._create_cursor(partition_state)
+
+            self._cursor_per_partition[
+                self._to_partition_key(record.associated_slice.partition)
+            ] = cursor
         return self._get_cursor(record).should_be_synced(
             self._convert_record_to_cursor_record(record)
         )

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -22,7 +22,6 @@ from airbyte_cdk.models import (
     ConnectorSpecification,
     FailureType,
 )
-from airbyte_cdk.sources.declarative.checks import COMPONENTS_CHECKER_TYPE_MAPPING
 from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
@@ -108,7 +107,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         if "type" not in check:
             check["type"] = "CheckStream"
         check_stream = self._constructor.create_component(
-            COMPONENTS_CHECKER_TYPE_MAPPING[check["type"]],
+            CheckStreamModel,
             check,
             dict(),
             emit_connector_builder_messages=self._emit_connector_builder_messages,

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -52,15 +52,6 @@ class CheckStream(BaseModel):
     )
 
 
-class CheckDynamicStream(BaseModel):
-    type: Literal["CheckDynamicStream"]
-    stream_count: int = Field(
-        ...,
-        description="Numbers of the streams to try reading from when running a check operation.",
-        title="Stream Count",
-    )
-
-
 class ConcurrencyLevel(BaseModel):
     type: Optional[Literal["ConcurrencyLevel"]] = None
     default_concurrency: Union[int, str] = Field(
@@ -490,24 +481,12 @@ class RefreshTokenUpdater(BaseModel):
 
 class OAuthAuthenticator(BaseModel):
     type: Literal["OAuthAuthenticator"]
-    client_id_name: Optional[str] = Field(
-        "client_id",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_app_id"],
-        title="Client ID Property Name",
-    )
     client_id: str = Field(
         ...,
         description="The OAuth client ID. Fill it in the user inputs.",
         examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"],
         title="Client ID",
     )
-    client_secret_name: Optional[str] = Field(
-        "client_secret",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_app_secret"],
-        title="Client Secret Property Name",
-    )
     client_secret: str = Field(
         ...,
         description="The OAuth client secret. Fill it in the user inputs.",
@@ -517,12 +496,6 @@ class OAuthAuthenticator(BaseModel):
         ],
         title="Client Secret",
     )
-    refresh_token_name: Optional[str] = Field(
-        "refresh_token",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_app_refresh_value"],
-        title="Refresh Token Property Name",
-    )
     refresh_token: Optional[str] = Field(
         None,
         description="Credential artifact used to get a new access token.",
@@ -556,12 +529,6 @@ class OAuthAuthenticator(BaseModel):
         examples=["expires_in"],
         title="Token Expiry Property Name",
     )
-    grant_type_name: Optional[str] = Field(
-        "grant_type",
-        description="The name of the property to use to refresh the `access_token`.",
-        examples=["custom_grant_type"],
-        title="Grant Type Property Name",
-    )
     grant_type: Optional[str] = Field(
         "refresh_token",
         description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
@@ -580,17 +547,6 @@ class OAuthAuthenticator(BaseModel):
         ],
         title="Refresh Request Body",
     )
-    refresh_request_headers: Optional[Dict[str, Any]] = Field(
-        None,
-        description="Headers of the request sent to get a new access token.",
-        examples=[
-            {
-                "Authorization": "<AUTH_TOKEN>",
-                "Content-Type": "application/x-www-form-urlencoded",
-            }
-        ],
-        title="Refresh Request Headers",
-    )
     scopes: Optional[List[str]] = Field(
         None,
         description="List of scopes that should be granted to the access token.",
@@ -781,43 +737,33 @@ class KeysToSnakeCase(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class FlattenFields(BaseModel):
-    type: Literal["FlattenFields"]
-    flatten_lists: Optional[bool] = Field(
-        True,
-        description="Whether to flatten lists or leave it as is. Default is True.",
-        title="Flatten Lists",
-    )
-    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
-
-
 class KeysReplace(BaseModel):
     type: Literal["KeysReplace"]
     old: str = Field(
         ...,
         description="Old value to replace.",
-        examples=[
-            " ",
-            "{{ record.id }}",
-            "{{ config['id'] }}",
-            "{{ stream_slice['id'] }}",
-        ],
+        examples=[" ", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
         title="Old value",
     )
     new: str = Field(
         ...,
         description="New value to set.",
-        examples=[
-            "_",
-            "{{ record.id }}",
-            "{{ config['id'] }}",
-            "{{ stream_slice['id'] }}",
-        ],
+        examples=["_", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
         title="New value",
     )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class FlattenFields(BaseModel):
+    type: Literal["FlattenFields"]
+    flatten_lists: Optional[bool] = Field(
+        True,
+        description="Whether to flatten lists or leave it as is. Default is True.",
+        title="Flatten Lists",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class IterableDecoder(BaseModel):
     type: Literal["IterableDecoder"]
 
@@ -903,8 +849,8 @@ class OauthConnectorInputSpecification(BaseModel):
         ...,
         description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",
-            "https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}",
+            "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
+            "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}",
         ],
         title="Consent URL",
     )
@@ -918,18 +864,14 @@ class OauthConnectorInputSpecification(BaseModel):
         ...,
         description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
         examples=[
-            "https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}"
+            "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
         ],
         title="Access Token URL",
     )
     access_token_headers: Optional[Dict[str, Any]] = Field(
         None,
         description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
-        examples=[
-            {
-                "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}"
-            }
-        ],
+        examples=[{"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"}],
         title="Access Token Headers",
     )
     access_token_params: Optional[Dict[str, Any]] = Field(
@@ -937,15 +879,15 @@ class OauthConnectorInputSpecification(BaseModel):
         description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
         examples=[
             {
-                "{{ auth_code_key }}": "{{ auth_code_value }}",
-                "{{ client_id_key }}": "{{ client_id_value }}",
-                "{{ client_secret_key }}": "{{ client_secret_value }}",
+                "{auth_code_key}": "{{auth_code_key}}",
+                "{client_id_key}": "{{client_id_key}}",
+                "{client_secret_key}": "{{client_secret_key}}",
             }
         ],
         title="Access Token Query Params (Json Encoded)",
     )
-    extract_output: Optional[List[str]] = Field(
-        None,
+    extract_output: List[str] = Field(
+        ...,
         description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.",
         examples=[["access_token", "refresh_token", "other_field"]],
         title="Extract Output",
@@ -1014,7 +956,7 @@ class OAuthConfigSpecification(BaseModel):
     )
     oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
         None,
-        description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n  + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n  + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n  + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{{ auth_code_key }}": "{{ auth_code_value }}",\n              "{{ client_id_key }}": "{{ client_id_value }}",\n              "{{ client_secret_key }}": "{{ client_secret_value }}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
+        description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n  + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n  + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n  + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{auth_code_key}": "{{auth_code_key}}",\n              "{client_id_key}": "{{client_id_key}}",\n              "{client_secret_key}": "{{client_secret_key}}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
         title="DeclarativeOAuth Connector Specification",
     )
     complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
@@ -1221,14 +1163,6 @@ class LegacySessionTokenAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class JsonParser(BaseModel):
-    class Config:
-        extra = Extra.allow
-
-    type: Literal["JsonParser"]
-    encoding: Optional[str] = "utf-8"
-
-
 class JsonLineParser(BaseModel):
     type: Literal["JsonLineParser"]
     encoding: Optional[str] = "utf-8"
@@ -1627,7 +1561,7 @@ class RecordSelector(BaseModel):
 
 class GzipParser(BaseModel):
     type: Literal["GzipParser"]
-    inner_parser: Union[JsonLineParser, CsvParser, JsonParser]
+    inner_parser: Union[JsonLineParser, CsvParser]
 
 
 class Spec(BaseModel):
@@ -1662,7 +1596,7 @@ class CompositeErrorHandler(BaseModel):
 
 class CompositeRawDecoder(BaseModel):
     type: Literal["CompositeRawDecoder"]
-    parser: Union[GzipParser, JsonParser, JsonLineParser, CsvParser]
+    parser: Union[GzipParser, JsonLineParser, CsvParser]
 
 
 class DeclarativeSource1(BaseModel):
@@ -1670,7 +1604,7 @@ class DeclarativeSource1(BaseModel):
         extra = Extra.forbid
 
     type: Literal["DeclarativeSource"]
-    check: Union[CheckStream, CheckDynamicStream]
+    check: CheckStream
     streams: List[DeclarativeStream]
     dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None
     version: str = Field(
@@ -1696,7 +1630,7 @@ class DeclarativeSource2(BaseModel):
         extra = Extra.forbid
 
     type: Literal["DeclarativeSource"]
-    check: Union[CheckStream, CheckDynamicStream]
+    check: CheckStream
     streams: Optional[List[DeclarativeStream]] = None
     dynamic_streams: List[DynamicDeclarativeStream]
     version: str = Field(
@@ -2106,10 +2040,6 @@ class AsyncRetriever(BaseModel):
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.",
     )
-    url_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
-        None,
-        description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.",
-    )
     download_requester: Union[CustomRequester, HttpRequester] = Field(
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.",