airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -7,8 +7,12 @@ version: 1.0.0
 required:
   - type
   - check
-  - streams
   - version
+anyOf:
+  - required:
+      - streams
+  - required:
+      - dynamic_streams
 properties:
   type:
     type: string
@@ -19,18 +23,28 @@ properties:
     type: array
     items:
       "$ref": "#/definitions/DeclarativeStream"
+  dynamic_streams:
+    type: array
+    items:
+      "$ref": "#/definitions/DynamicDeclarativeStream"
   version:
     type: string
+    description: The version of the Airbyte CDK used to build and test the source.
   schemas:
     "$ref": "#/definitions/Schemas"
   definitions:
     type: object
   spec:
     "$ref": "#/definitions/Spec"
+  concurrency_level:
+    "$ref": "#/definitions/ConcurrencyLevel"
   metadata:
     type: object
     description: For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.
     additionalProperties: true
+  description:
+    type: string
+    description: A description of the connector. It will be presented on the Source documentation page.
 additionalProperties: false
 definitions:
   AddedFieldDefinition:
@@ -257,6 +271,7 @@ definitions:
             - "$ref": "#/definitions/BearerAuthenticator"
             - "$ref": "#/definitions/CustomAuthenticator"
             - "$ref": "#/definitions/OAuthAuthenticator"
+            - "$ref": "#/definitions/JwtAuthenticator"
             - "$ref": "#/definitions/NoAuth"
             - "$ref": "#/definitions/SessionTokenAuthenticator"
             - "$ref": "#/definitions/LegacySessionTokenAuthenticator"
@@ -264,6 +279,7 @@ definitions:
           - authenticators:
               token: "#/definitions/ApiKeyAuthenticator"
               oauth: "#/definitions/OAuthAuthenticator"
+              jwt: "#/definitions/JwtAuthenticator"
       $parameters:
         type: object
         additionalProperties: true
@@ -309,6 +325,37 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  ConcurrencyLevel:
+    title: Concurrency Level
+    description: Defines the amount of parallelization for the streams that are being synced. The factor of parallelization is how many partitions or streams are synced at the same time. For example, with a concurrency_level of 10, ten streams or partitions of data will processed at the same time. Note that a value of 1 could create deadlock if a stream has a very high number of partitions.
+    type: object
+    required:
+      - default_concurrency
+    properties:
+      type:
+        type: string
+        enum: [ConcurrencyLevel]
+      default_concurrency:
+        title: Default Concurrency
+        description: The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.
+        anyOf:
+          - type: integer
+          - type: string
+        interpolation_context:
+          - config
+        examples:
+          - 10
+          - "{{ config['num_workers'] or 10 }}"
+      max_concurrency:
+        title: Max Concurrency
+        description: The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.
+        type: integer
+        examples:
+          - 20
+          - 100
+      $parameters:
+        type: object
+        additionalProperties: true
   ConstantBackoffStrategy:
     title: Constant Backoff
     description: Backoff strategy with a constant backoff interval.
@@ -353,11 +400,12 @@ definitions:
         interpolation_context:
           - config
           - headers
-          - last_records
+          - last_page_size
+          - last_record
           - response
         examples:
           - "{{ headers.link.next.cursor }}"
-          - "{{ last_records[-1]['key'] }}"
+          - "{{ last_record['key'] }}"
           - "{{ response['nextPage'] }}"
       page_size:
         title: Page Size
@@ -372,15 +420,11 @@ definitions:
         interpolation_context:
           - config
           - headers
-          - last_records
+          - last_record
           - response
         examples:
           - "{{ response.data.has_more is false }}"
           - "{{ 'next' not in headers['link'] }}"
-      decoder:
-        title: Decoder
-        description: Component decoding the response so records can be extracted.
-        "$ref": "#/definitions/JsonDecoder"
       $parameters:
         type: object
         additionalProperties: true
@@ -623,6 +667,49 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  CustomSchemaNormalization:
+    title: Custom Schema Normalization
+    description: Schema normalization component whose behavior is derived from a custom code implementation of the connector.
+    type: object
+    additionalProperties: true
+    required:
+      - type
+      - class_name
+    properties:
+      type:
+        type: string
+        enum: [ CustomSchemaNormalization ]
+      class_name:
+        title: Class Name
+        description: Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.
+        type: string
+        additionalProperties: true
+        examples:
+          - "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer"
+      $parameters:
+        type: object
+        additionalProperties: true
+  CustomStateMigration:
+    title: Custom State Migration
+    description: Apply a custom transformation on the input state.
+    type: object
+    additionalProperties: true
+    required:
+      - type
+      - class_name
+    properties:
+      type:
+        type: string
+        enum: [CustomStateMigration]
+      class_name:
+        title: Class Name
+        description: Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.
+        type: string
+        examples:
+          - "source_railz.components.MyCustomStateMigration"
+      $parameters:
+        type: object
+        additionalProperties: true
   CustomTransformation:
     title: Custom Transformation
     description: Transformation component whose behavior is derived from a custom code implementation of the connector.
@@ -644,6 +731,29 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  LegacyToPerPartitionStateMigration:
+    title: Legacy To Per-partition-state Migration
+    description:
+      'Transforms the input state for per-partitioned streams from the legacy format to the low-code format.
+      The cursor field and partition ID fields are automatically extracted from the stream''s DatetimebasedCursor and SubstreamPartitionRouter.
+
+      Example input state:
+      {
+      "13506132": {
+      "last_changed": "2022-12-27T08:34:39+00:00"
+      }
+      Example output state:
+      {
+      "partition": {"id": "13506132"},
+      "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}
+      }
+      '
+    type: object
+    additionalProperties: true
+    properties:
+      type:
+        type: string
+        enum: [LegacyToPerPartitionStateMigration]
   DatetimeBasedCursor:
     title: Datetime Based Cursor
     description: Cursor to provide incremental capabilities over datetime.
@@ -671,6 +781,7 @@ definitions:
         description: |
           The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
             * **%s**: Epoch unix timestamp - `1686218963`
+            * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`
             * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`
             * **%a**: Weekday (abbreviated) - `Sun`
             * **%A**: Weekday (full) - `Sunday`
@@ -704,6 +815,7 @@ definitions:
           - "%Y-%m-%d"
           - "%s"
           - "%ms"
+          - "%s_as_float"
       start_datetime:
         title: Start Datetime
         description: The datetime that determines the earliest record that should be synced.
@@ -753,6 +865,20 @@ definitions:
         title: Whether the target API is formatted as a data feed
         description: A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.
         type: boolean
+      is_client_side_incremental:
+        title: Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)
+        description: If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.
+        type: boolean
+      is_compare_strictly:
+        title: Whether to skip requests if the start time equals the end time
+        description: Set to True if the target API does not accept queries where the start time equal the end time.
+        type: boolean
+        default: False
+      global_substream_cursor:
+        title: Whether to store cursor as one value instead of per partition
+        description: This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).
+        type: boolean
+        default: false
       lookback_window:
         title: Lookback Window
         description: Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.
@@ -788,6 +914,127 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  JwtAuthenticator:
+    title: JWT Authenticator
+    description: Authenticator for requests using JWT authentication flow.
+    type: object
+    required:
+      - type
+      - secret_key
+      - algorithm
+    properties:
+      type:
+        type: string
+        enum: [JwtAuthenticator]
+      secret_key:
+        type: string
+        description: Secret used to sign the JSON web token.
+        examples:
+          - "{{ config['secret_key'] }}"
+      base64_encode_secret_key:
+        type: boolean
+        description: When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.
+        default: False
+      algorithm:
+        type: string
+        description: Algorithm used to sign the JSON web token.
+        enum:
+          [
+            "HS256",
+            "HS384",
+            "HS512",
+            "ES256",
+            "ES256K",
+            "ES384",
+            "ES512",
+            "RS256",
+            "RS384",
+            "RS512",
+            "PS256",
+            "PS384",
+            "PS512",
+            "EdDSA",
+          ]
+        examples:
+          - ES256
+          - HS256
+          - RS256
+          - "{{ config['algorithm'] }}"
+      token_duration:
+        type: integer
+        title: Token Duration
+        description: The amount of time in seconds a JWT token can be valid after being issued.
+        default: 1200
+        examples:
+          - 1200
+          - 3600
+      header_prefix:
+        type: string
+        title: Header Prefix
+        description: The prefix to be used within the Authentication header.
+        examples:
+          - "Bearer"
+          - "Basic"
+      jwt_headers:
+        type: object
+        title: JWT Headers
+        description: JWT headers used when signing JSON web token.
+        additionalProperties: false
+        properties:
+          kid:
+            type: string
+            title: Key Identifier
+            description: Private key ID for user account.
+            examples:
+              - "{{ config['kid'] }}"
+          typ:
+            type: string
+            title: Type
+            description: The media type of the complete JWT.
+            default: JWT
+            examples:
+              - JWT
+          cty:
+            type: string
+            title: Content Type
+            description: Content type of JWT header.
+            examples:
+              - JWT
+      additional_jwt_headers:
+        type: object
+        title: Additional JWT Headers
+        description: Additional headers to be included with the JWT headers object.
+        additionalProperties: true
+      jwt_payload:
+        type: object
+        title: JWT Payload
+        description: JWT Payload used when signing JSON web token.
+        additionalProperties: false
+        properties:
+          iss:
+            type: string
+            title: Issuer
+            description: The user/principal that issued the JWT. Commonly a value unique to the user.
+            examples:
+              - "{{ config['iss'] }}"
+          sub:
+            type: string
+            title: Subject
+            description: The subject of the JWT. Commonly defined by the API.
+          aud:
+            type: string
+            title: Audience
+            description: The recipient that the JWT is intended for. Commonly defined by the API.
+            examples:
+              - "appstoreconnect-v1"
+      additional_jwt_payload:
+        type: object
+        title: Additional JWT Payload Properties
+        description: Additional properties to be added to the JWT payload.
+        additionalProperties: true
+      $parameters:
+        type: object
+        additionalProperties: true
   OAuthAuthenticator:
     title: OAuth2
     description: Authenticator for requests using OAuth 2.0 authorization flow.
@@ -796,7 +1043,6 @@ definitions:
       - type
       - client_id
       - client_secret
-      - token_refresh_endpoint
     properties:
       type:
         type: string
@@ -835,6 +1081,12 @@ definitions:
         default: "access_token"
         examples:
           - access_token
+      access_token_value:
+        title: Access Token Value
+        description: The value of the access_token to bypass the token refreshing using `refresh_token`.
+        type: string
+        examples:
+          - secret_access_token_value
       expires_in_name:
         title: Token Expiry Property Name
         description: The name of the property which contains the expiry date in the response from the token refresh endpoint.
@@ -924,6 +1176,31 @@ definitions:
             default: ["credentials", "token_expiry_date"]
             examples:
               - ["credentials", "token_expiry_date"]
+          refresh_token_error_status_codes:
+            title: Refresh Token Error Status Codes
+            description: Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error
+            type: array
+            items:
+              type: integer
+            default: []
+            examples:
+              - [400, 500]
+          refresh_token_error_key:
+            title: Refresh Token Error Key
+            description: Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).
+            type: string
+            default: ""
+            examples:
+              - "error"
+          refresh_token_error_values:
+            title: Refresh Token Error Values
+            description: 'List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error'
+            type: array
+            items:
+              type: string
+            default: []
+            examples:
+              - ["invalid_grant", "invalid_permissions"]
       $parameters:
         type: object
         additionalProperties: true
@@ -943,6 +1220,7 @@ definitions:
         title: Retriever
         description: Component used to coordinate how records are extracted across stream slices and request pages.
         anyOf:
+          - "$ref": "#/definitions/AsyncRetriever"
           - "$ref": "#/definitions/CustomRetriever"
           - "$ref": "#/definitions/SimpleRetriever"
       incremental_sync:
@@ -967,6 +1245,7 @@ definitions:
         title: Schema Loader
         description: Component used to retrieve the schema for the current stream.
         anyOf:
+          - "$ref": "#/definitions/DynamicSchemaLoader"
           - "$ref": "#/definitions/InlineSchemaLoader"
           - "$ref": "#/definitions/JsonFileSchemaLoader"
           - "$ref": "#/definitions/CustomSchemaLoader"
@@ -981,6 +1260,19 @@ definitions:
             - "$ref": "#/definitions/AddFields"
             - "$ref": "#/definitions/CustomTransformation"
             - "$ref": "#/definitions/RemoveFields"
+            - "$ref": "#/definitions/KeysToLower"
+            - "$ref": "#/definitions/KeysToSnakeCase"
+            - "$ref": "#/definitions/FlattenFields"
+            - "$ref": "#/definitions/KeysReplace"
+      state_migrations:
+        title: State Migrations
+        description: Array of state migrations to be applied on the input state
+        type: array
+        items:
+          anyOf:
+            - "$ref": "#/definitions/LegacyToPerPartitionStateMigration"
+            - "$ref": "#/definitions/CustomStateMigration"
+        default: []
       $parameters:
         type: object
         additional_properties: true
@@ -1042,10 +1334,6 @@ definitions:
           - "$ref": "#/definitions/CustomPaginationStrategy"
           - "$ref": "#/definitions/OffsetIncrement"
           - "$ref": "#/definitions/PageIncrement"
-      decoder:
-        title: Decoder
-        description: Component decoding the response so records can be extracted.
-        "$ref": "#/definitions/JsonDecoder"
       page_size_option:
         "$ref": "#/definitions/RequestOption"
       page_token_option:
@@ -1072,17 +1360,26 @@ definitions:
         type: array
         items:
           - type: string
-        interpolation_content:
+        interpolation_context:
           - config
         examples:
           - ["data"]
           - ["data", "records"]
           - ["data", "{{ parameters.name }}"]
           - ["data", "*", "record"]
-      decoder:
-        title: Decoder
-        description: Component decoding the response so records can be extracted.
-        "$ref": "#/definitions/JsonDecoder"
+      $parameters:
+        type: object
+        additionalProperties: true
+  ResponseToFileExtractor:
+    title: CSV To File Extractor
+    description: A record extractor designed for handling large responses that may exceed memory limits (to prevent OOM issues). It downloads a CSV file to disk, reads the data from disk, and deletes the file once it has been fully processed.
+    type: object
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [ResponseToFileExtractor]
       $parameters:
         type: object
         additionalProperties: true
@@ -1157,6 +1454,12 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/SessionTokenRequestApiKeyAuthenticator"
           - "$ref": "#/definitions/SessionTokenRequestBearerAuthenticator"
+      decoder:
+        title: Decoder
+        description: Component used to decode the response.
+        anyOf:
+          - "$ref": "#/definitions/JsonDecoder"
+          - "$ref": "#/definitions/XmlDecoder"
       $parameters:
         type: object
         additionalProperties: true
@@ -1232,6 +1535,7 @@ definitions:
           - "$ref": "#/definitions/BearerAuthenticator"
           - "$ref": "#/definitions/CustomAuthenticator"
           - "$ref": "#/definitions/OAuthAuthenticator"
+          - "$ref": "#/definitions/JwtAuthenticator"
           - "$ref": "#/definitions/NoAuth"
           - "$ref": "#/definitions/SessionTokenAuthenticator"
           - "$ref": "#/definitions/LegacySessionTokenAuthenticator"
@@ -1342,7 +1646,6 @@ definitions:
     type: object
     required:
       - type
-      - action
     properties:
       type:
         type: string
@@ -1356,11 +1659,25 @@ definitions:
           - FAIL
           - RETRY
           - IGNORE
+          - RATE_LIMITED
         examples:
           - SUCCESS
           - FAIL
           - RETRY
           - IGNORE
+          - RATE_LIMITED
+      failure_type:
+        title: Failure Type
+        description: Failure type of traced exception if a response matches the filter.
+        type: string
+        enum:
+          - system_error
+          - config_error
+          - transient_error
+        examples:
+          - system_error
+          - config_error
+          - transient_error
       error_message:
         title: Error Message
         description: Error Message to display if the response matches the filter.
@@ -1398,6 +1715,105 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  TypesMap:
+    title: Types Map
+    description: (This component is experimental. Use at your own risk.) Represents a mapping between a current type and its corresponding target type.
+    type: object
+    required:
+      - target_type
+      - current_type
+    properties:
+      target_type:
+        anyOf:
+          - type: string
+          - type: array
+            items:
+              type: string
+      current_type:
+        anyOf:
+          - type: string
+          - type: array
+            items:
+              type: string
+  SchemaTypeIdentifier:
+    title: Schema Type Identifier
+    description: (This component is experimental. Use at your own risk.) Identifies schema details for dynamic schema extraction and processing.
+    type: object
+    required:
+      - key_pointer
+    properties:
+      type:
+        type: string
+        enum: [SchemaTypeIdentifier]
+      schema_pointer:
+        title: Schema Path
+        description: List of nested fields defining the schema field path to extract. Defaults to [].
+        type: array
+        default: []
+        items:
+          - type: string
+        interpolation_context:
+          - config
+      key_pointer:
+        title: Key Path
+        description: List of potentially nested fields describing the full path of the field key to extract.
+        type: array
+        items:
+          - type: string
+        interpolation_context:
+          - config
+      type_pointer:
+        title: Type Path
+        description: List of potentially nested fields describing the full path of the field type to extract.
+        type: array
+        items:
+          - type: string
+        interpolation_context:
+          - config
+      types_mapping:
+        type: array
+        items:
+          - "$ref": "#/definitions/TypesMap"
+      $parameters:
+        type: object
+        additionalProperties: true
+  DynamicSchemaLoader:
+    title: Dynamic Schema Loader
+    description: (This component is experimental. Use at your own risk.) Loads a schema by extracting data from retrieved records.
+    type: object
+    required:
+      - type
+      - retriever
+      - schema_type_identifier
+    properties:
+      type:
+        type: string
+        enum: [DynamicSchemaLoader]
+      retriever:
+        title: Retriever
+        description: Component used to coordinate how records are extracted across stream slices and request pages.
+        anyOf:
+          - "$ref": "#/definitions/AsyncRetriever"
+          - "$ref": "#/definitions/CustomRetriever"
+          - "$ref": "#/definitions/SimpleRetriever"
+      schema_transformations:
+        title: Schema Transformations
+        description: A list of transformations to be applied to the schema.
+        type: array
+        items:
+          anyOf:
+            - "$ref": "#/definitions/AddFields"
+            - "$ref": "#/definitions/CustomTransformation"
+            - "$ref": "#/definitions/RemoveFields"
+            - "$ref": "#/definitions/KeysToLower"
+            - "$ref": "#/definitions/KeysToSnakeCase"
+            - "$ref": "#/definitions/FlattenFields"
+            - "$ref": "#/definitions/KeysReplace"
+      schema_type_identifier:
+        "$ref": "#/definitions/SchemaTypeIdentifier"
+      $parameters:
+        type: object
+        additionalProperties: true
   InlineSchemaLoader:
     title: Inline Schema Loader
     description: Loads a schema that is defined directly in the manifest file.
@@ -1442,66 +1858,222 @@ definitions:
       type:
         type: string
         enum: [JsonDecoder]
-  ListPartitionRouter:
-    title: List Partition Router
-    description: A Partition router that specifies a list of attributes where each attribute describes a portion of the complete data set for a stream. During a sync, each value is iterated over and can be used as input to outbound API requests.
+  JsonlDecoder:
+    title: JSONL Decoder
+    description: Use this if the response consists of JSON objects separated by new lines (`\n`) in JSONL format.
     type: object
     required:
       - type
-      - cursor_field
-      - values
     properties:
       type:
         type: string
-        enum: [ListPartitionRouter]
-      cursor_field:
-        title: Current Partition Value Identifier
-        description: While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition['my_key'] }}" where "my_key" is the value of the cursor_field.
+        enum: [JsonlDecoder]
+  KeysToLower:
+    title: Keys to Lower Case
+    description: A transformation that renames all keys to lower case.
+    type: object
+    required:
+      - type
+    properties:
+      type:
         type: string
-        interpolation_context:
-          - config
-        examples:
-          - "section"
-          - "{{ config['section_key'] }}"
-      values:
-        title: Partition Values
-        description: The list of attributes being iterated over and used as input for the requests made to the source API.
-        anyOf:
-          - type: string
-          - type: array
-            items:
-              type: string
-        interpolation_context:
-          - config
-        examples:
-          - ["section_a", "section_b", "section_c"]
-          - "{{ config['sections'] }}"
-      request_option:
-        title: Inject Partition Value Into Outgoing HTTP Request
-        description: A request option describing where the list value should be injected into and under what field name if applicable.
-        "$ref": "#/definitions/RequestOption"
+        enum: [KeysToLower]
       $parameters:
         type: object
         additionalProperties: true
-  MinMaxDatetime:
-    title: Min-Max Datetime
-    description: Compares the provided date against optional minimum or maximum times. The max_datetime serves as the ceiling and will be returned when datetime exceeds it. The min_datetime serves as the floor.
+  KeysToSnakeCase:
+    title: Key to Snake Case
+    description: A transformation that renames all keys to snake case.
     type: object
     required:
       - type
-      - datetime
     properties:
       type:
         type: string
-        enum: [MinMaxDatetime]
-      datetime:
-        title: Datetime
-        description: Datetime value.
+        enum: [KeysToSnakeCase]
+      $parameters:
+        type: object
+        additionalProperties: true
+  FlattenFields:
+    title: Flatten Fields
+    description: A transformation that flatten record to single level format.
+    type: object
+    required:
+      - type
+    properties:
+      type:
         type: string
-        interpolation_context:
-          - config
-        examples:
-          - 2021-01-01
+        enum: [FlattenFields]
+      flatten_lists:
+        title: Flatten Lists
+        description: Whether to flatten lists or leave it as is. Default is True.
+        type: boolean
+        default: true
+      $parameters:
+        type: object
+        additionalProperties: true
+  KeysReplace:
+    title: Keys Replace
+    description: A transformation that replaces symbols in keys.
+    type: object
+    required:
+      - type
+      - old
+      - new
+    properties:
+      type:
+        type: string
+        enum: [KeysReplace]
+      old:
+        type: string
+        title: Old value
+        description: Old value to replace.
+        examples:
+          - " "
+          - "{{ record.id }}"
+          - "{{ config['id'] }}"
+          - "{{ stream_slice['id'] }}"
+        interpolation_context:
+          - config
+          - record
+          - stream_state
+          - stream_slice
+      new:
+        type: string
+        title: New value
+        description: New value to set.
+        examples:
+          - "_"
+          - "{{ record.id }}"
+          - "{{ config['id'] }}"
+          - "{{ stream_slice['id'] }}"
+        interpolation_context:
+          - config
+          - record
+          - stream_state
+          - stream_slice
+      $parameters:
+        type: object
+        additionalProperties: true
+  IterableDecoder:
+    title: Iterable Decoder
+    description: Use this if the response consists of strings separated by new lines (`\n`). The Decoder will wrap each row into a JSON object with the `record` key.
+    type: object
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [IterableDecoder]
+  XmlDecoder:
+    title: XML Decoder
+    description: Use this if the response is XML.
+    type: object
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [XmlDecoder]
+  CustomDecoder:
+    title: Custom Decoder
+    description: Use this to implement custom decoder logic.
+    type: object
+    additionalProperties: true
+    required:
+      - type
+      - class_name
+    properties:
+      type:
+        type: string
+        enum: [CustomDecoder]
+      class_name:
+        title: Class Name
+        description: Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.
+        type: string
+        additionalProperties: true
+        examples:
+          - "source_amazon_ads.components.GzipJsonlDecoder"
+      $parameters:
+        type: object
+        additionalProperties: true
+  GzipJsonDecoder:
+    title: GzipJson Decoder
+    description: Use this if the response is Gzip compressed Json.
+    type: object
+    additionalProperties: true
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [GzipJsonDecoder]
+      encoding:
+        type: string
+        default: utf-8
+      $parameters:
+        type: object
+        additionalProperties: true
+  ListPartitionRouter:
+    title: List Partition Router
+    description: A Partition router that specifies a list of attributes where each attribute describes a portion of the complete data set for a stream. During a sync, each value is iterated over and can be used as input to outbound API requests.
+    type: object
+    required:
+      - type
+      - cursor_field
+      - values
+    properties:
+      type:
+        type: string
+        enum: [ListPartitionRouter]
+      cursor_field:
+        title: Current Partition Value Identifier
+        description: While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition['my_key'] }}" where "my_key" is the value of the cursor_field.
+        type: string
+        interpolation_context:
+          - config
+        examples:
+          - "section"
+          - "{{ config['section_key'] }}"
+      values:
+        title: Partition Values
+        description: The list of attributes being iterated over and used as input for the requests made to the source API.
+        anyOf:
+          - type: string
+          - type: array
+            items:
+              type: string
+        interpolation_context:
+          - config
+        examples:
+          - ["section_a", "section_b", "section_c"]
+          - "{{ config['sections'] }}"
+      request_option:
+        title: Inject Partition Value Into Outgoing HTTP Request
+        description: A request option describing where the list value should be injected into and under what field name if applicable.
+        "$ref": "#/definitions/RequestOption"
+      $parameters:
+        type: object
+        additionalProperties: true
+  MinMaxDatetime:
+    title: Min-Max Datetime
+    description: Compares the provided date against optional minimum or maximum times. The max_datetime serves as the ceiling and will be returned when datetime exceeds it. The min_datetime serves as the floor.
+    type: object
+    required:
+      - type
+      - datetime
+    properties:
+      type:
+        type: string
+        enum: [MinMaxDatetime]
+      datetime:
+        title: Datetime
+        description: Datetime value.
+        type: string
+        interpolation_context:
+          - config
+        examples:
+          - 2021-01-01
           - 2021-01-01T00:00:00Z
           - "{{ config['start_time'] }}"
       datetime_format:
@@ -1509,6 +2081,7 @@ definitions:
         description: |
           Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
             * **%s**: Epoch unix timestamp - `1686218963`
+            * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`
             * **%ms**: Epoch unix timestamp - `1686218963123`
             * **%a**: Weekday (abbreviated) - `Sun`
             * **%A**: Weekday (full) - `Sunday`
@@ -1624,6 +2197,170 @@ definitions:
           - app_id:
               type: string
               path_in_connector_config: ["info", "app_id"]
+      oauth_connector_input_specification:
+        title: DeclarativeOAuth Connector Specification
+        description: |-
+          The DeclarativeOAuth specific blob.
+          Pertains to the fields defined by the connector relating to the OAuth flow.
+
+          Interpolation capabilities:
+          - The variables placeholders are declared as `{my_var}`.
+          - The nested resolution variables like `{{my_nested_var}}` is allowed as well.
+
+          - The allowed interpolation context is:
+            + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}
+            + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}
+            + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}
+            + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}
+            + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}
+
+          Examples:
+            - The TikTok Marketing DeclarativeOAuth spec:
+            {
+              "oauth_connector_input_specification": {
+                "type": "object",
+                "additionalProperties": false,
+                "properties": {
+                    "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
+                    "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",
+                    "access_token_params": {
+                        "{auth_code_key}": "{{auth_code_key}}",
+                        "{client_id_key}": "{{client_id_key}}",
+                        "{client_secret_key}": "{{client_secret_key}}"
+                    },
+                    "access_token_headers": {
+                        "Content-Type": "application/json",
+                        "Accept": "application/json"
+                    },
+                    "extract_output": ["data.access_token"],
+                    "client_id_key": "app_id",
+                    "client_secret_key": "secret",
+                    "auth_code_key": "auth_code"
+                }
+              }
+            }
+        type: object
+        additionalProperties: true
+        required:
+          - consent_url
+          - access_token_url
+          - extract_output
+        properties:
+          consent_url:
+            title: Consent URL
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific string URL string template to initiate the authentication.
+              The placeholders are replaced during the processing to provide neccessary values.
+            examples:
+              - https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}
+              - https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}
+          scope:
+            title: Scopes
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.
+            examples:
+              - user:read user:read_orders workspaces:read
+          access_token_url:
+            title: Access Token URL
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.
+              The placeholders are replaced during the processing to provide neccessary values.
+            examples:
+              - https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}
+          access_token_headers:
+            title: Access Token Headers
+            type: object
+            additionalProperties: true
+            description: |-
+              The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
+            examples:
+              - {
+                  "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}",
+                }
+          access_token_params:
+            title: Access Token Query Params (Json Encoded)
+            type: object
+            additionalProperties: true
+            description: |-
+              The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
+              When this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.
+            examples:
+              - {
+                  "{auth_code_key}": "{{auth_code_key}}",
+                  "{client_id_key}": "{{client_id_key}}",
+                  "{client_secret_key}": "{{client_secret_key}}",
+                }
+          extract_output:
+            title: Extract Output
+            type: array
+            items:
+              type: string
+            description: |-
+              The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.
+            examples:
+              - ["access_token", "refresh_token", "other_field"]
+          state:
+            title: Configurable State Query Param
+            type: object
+            additionalProperties: true
+            required:
+              - min
+              - max
+            description: |-
+              The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,
+              including length and complexity.
+            properties:
+              min:
+                type: integer
+              max:
+                type: integer
+            examples:
+              - { "min": 7, "max": 128 }
+          client_id_key:
+            title: Client ID Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.
+            examples:
+              - "my_custom_client_id_key_name"
+          client_secret_key:
+            title: Client Secret Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.
+            examples:
+              - "my_custom_client_secret_key_name"
+          scope_key:
+            title: Scopes Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.
+            examples:
+              - "my_custom_scope_key_key_name"
+          state_key:
+            title: State Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.
+            examples:
+              - "my_custom_state_key_key_name"
+          auth_code_key:
+            title: Auth Code Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.
+            examples:
+              - "my_custom_auth_code_key_name"
+          redirect_uri_key:
+            title: Redirect URI Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.
+            examples:
+              - "my_custom_redirect_uri_key_name"
       complete_oauth_output_specification:
         title: "OAuth output specification"
         description: |-
@@ -1801,6 +2538,25 @@ definitions:
         title: Request Option
         description: A request option describing where the parent key value should be injected into and under what field name if applicable.
         "$ref": "#/definitions/RequestOption"
+      incremental_dependency:
+        title: Incremental Dependency
+        description: Indicates whether the parent stream should be read incrementally based on updates in the child stream.
+        type: boolean
+        default: false
+      extra_fields:
+        title: Extra Fields
+        description: Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.
+        interpolation_context:
+          - config
+        type: array
+        items:
+          type: array
+          items:
+            type: string
+          description: Defines a field path as an array of strings.
+          examples:
+            - ["field1"]
+            - ["nested", "field2"]
       $parameters:
         type: object
         additionalProperties: true
@@ -1871,7 +2627,11 @@ definitions:
           - "$ref": "#/definitions/CustomRecordFilter"
           - "$ref": "#/definitions/RecordFilter"
       schema_normalization:
-        "$ref": "#/definitions/SchemaNormalization"
+        title: Schema Normalization
+        description: Responsible for normalization according to the schema.
+        anyOf:
+          - "$ref": "#/definitions/SchemaNormalization"
+          - "$ref": "#/definitions/CustomSchemaNormalization"
         default: None
       $parameters:
         type: object
@@ -2072,6 +2832,205 @@ definitions:
                 - "$ref": "#/definitions/CustomPartitionRouter"
                 - "$ref": "#/definitions/ListPartitionRouter"
                 - "$ref": "#/definitions/SubstreamPartitionRouter"
+      decoder:
+        title: Decoder
+        description: Component decoding the response so records can be extracted.
+        anyOf:
+          - "$ref": "#/definitions/CustomDecoder"
+          - "$ref": "#/definitions/JsonDecoder"
+          - "$ref": "#/definitions/JsonlDecoder"
+          - "$ref": "#/definitions/IterableDecoder"
+          - "$ref": "#/definitions/XmlDecoder"
+          - "$ref": "#/definitions/GzipJsonDecoder"
+          - "$ref": "#/definitions/CompositeRawDecoder"
+      $parameters:
+        type: object
+        additionalProperties: true
+  CompositeRawDecoder:
+    description: "(This is experimental, use at your own risk)"
+    type: object
+    required:
+      - type
+      - parser
+    properties:
+      type:
+        type: string
+        enum: [CompositeRawDecoder]
+      parser:
+        anyOf:
+          - "$ref": "#/definitions/GzipParser"
+          - "$ref": "#/definitions/JsonLineParser"
+          - "$ref": "#/definitions/CsvParser"
+  #    PARSERS
+  GzipParser:
+    type: object
+    required:
+      - type
+      - inner_parser
+    properties:
+      type:
+        type: string
+        enum: [GzipParser]
+      inner_parser:
+        anyOf:
+          - "$ref": "#/definitions/JsonLineParser"
+          - "$ref": "#/definitions/CsvParser"
+  JsonLineParser:
+    type: object
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [JsonLineParser]
+      encoding:
+        type: string
+        default: utf-8
+  CsvParser:
+    type: object
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [CsvParser]
+      encoding:
+        type: string
+        default: utf-8
+      delimiter:
+        type: string
+        default: ","
+  AsyncJobStatusMap:
+    description: Matches the api job status to Async Job Status.
+    type: object
+    required:
+      - running
+      - completed
+      - failed
+      - timeout
+    properties:
+      type:
+        type: string
+        enum: [AsyncJobStatusMap]
+      running:
+        type: array
+        items:
+          type: string
+      completed:
+        type: array
+        items:
+          type: string
+      failed:
+        type: array
+        items:
+          type: string
+      timeout:
+        type: array
+        items:
+          type: string
+  AsyncRetriever:
+    description: "[Experimental - We expect the interface to change shortly and we reserve the right to not consider this a breaking change] Retrieves records by Asynchronously sending requests to fetch records. The retriever acts as an orchestrator between the requester, the record selector, the paginator, and the partition router."
+    type: object
+    required:
+      - type
+      - record_selector
+      - status_mapping
+      - creation_requester
+      - polling_requester
+      - download_requester
+      - status_extractor
+      - urls_extractor
+    properties:
+      type:
+        type: string
+        enum: [AsyncRetriever]
+      record_selector:
+        description: Component that describes how to extract records from a HTTP response.
+        "$ref": "#/definitions/RecordSelector"
+      status_mapping:
+        description: Async Job Status to Airbyte CDK Async Job Status mapping.
+        anyOf:
+          - "$ref": "#/definitions/AsyncJobStatusMap"
+      status_extractor:
+        description: Responsible for fetching the actual status of the async job.
+        anyOf:
+          - "$ref": "#/definitions/CustomRecordExtractor"
+          - "$ref": "#/definitions/DpathExtractor"
+      urls_extractor:
+        description: Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.
+        anyOf:
+          - "$ref": "#/definitions/CustomRecordExtractor"
+          - "$ref": "#/definitions/DpathExtractor"
+      download_extractor:
+        description: Responsible for fetching the records from provided urls.
+        anyOf:
+          - "$ref": "#/definitions/CustomRecordExtractor"
+          - "$ref": "#/definitions/DpathExtractor"
+          - "$ref": "#/definitions/ResponseToFileExtractor"
+      creation_requester:
+        description: Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.
+        anyOf:
+          - "$ref": "#/definitions/CustomRequester"
+          - "$ref": "#/definitions/HttpRequester"
+      polling_requester:
+        description: Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.
+        anyOf:
+          - "$ref": "#/definitions/CustomRequester"
+          - "$ref": "#/definitions/HttpRequester"
+      download_requester:
+        description: Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.
+        anyOf:
+          - "$ref": "#/definitions/CustomRequester"
+          - "$ref": "#/definitions/HttpRequester"
+      download_paginator:
+        description: Paginator component that describes how to navigate through the API's pages during download.
+        anyOf:
+          - "$ref": "#/definitions/DefaultPaginator"
+          - "$ref": "#/definitions/NoPagination"
+      abort_requester:
+        description: Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.
+        anyOf:
+          - "$ref": "#/definitions/CustomRequester"
+          - "$ref": "#/definitions/HttpRequester"
+      delete_requester:
+        description: Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.
+        anyOf:
+          - "$ref": "#/definitions/CustomRequester"
+          - "$ref": "#/definitions/HttpRequester"
+      partition_router:
+        title: Partition Router
+        description: PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.
+        default: []
+        anyOf:
+          - "$ref": "#/definitions/CustomPartitionRouter"
+          - "$ref": "#/definitions/ListPartitionRouter"
+          - "$ref": "#/definitions/SubstreamPartitionRouter"
+          - type: array
+            items:
+              anyOf:
+                - "$ref": "#/definitions/CustomPartitionRouter"
+                - "$ref": "#/definitions/ListPartitionRouter"
+                - "$ref": "#/definitions/SubstreamPartitionRouter"
+      decoder:
+        title: Decoder
+        description: Component decoding the response so records can be extracted.
+        anyOf:
+          - "$ref": "#/definitions/CustomDecoder"
+          - "$ref": "#/definitions/JsonDecoder"
+          - "$ref": "#/definitions/JsonlDecoder"
+          - "$ref": "#/definitions/IterableDecoder"
+          - "$ref": "#/definitions/XmlDecoder"
+          - "$ref": "#/definitions/GzipJsonDecoder"
+      download_decoder:
+        title: Download Decoder
+        description: Component decoding the download response so records can be extracted.
+        anyOf:
+          - "$ref": "#/definitions/CustomDecoder"
+          - "$ref": "#/definitions/JsonDecoder"
+          - "$ref": "#/definitions/JsonlDecoder"
+          - "$ref": "#/definitions/IterableDecoder"
+          - "$ref": "#/definitions/XmlDecoder"
+          - "$ref": "#/definitions/GzipJsonDecoder"
       $parameters:
         type: object
         additionalProperties: true
@@ -2155,6 +3114,12 @@ definitions:
         type: string
         examples:
           - "([-+]?\\d+)"
+      max_waiting_time_in_seconds:
+        title: Max Waiting Time in Seconds
+        description: Given the value extracted from the header is greater than this value, stop the stream.
+        type: number
+        examples:
+          - 3600
       $parameters:
         type: object
         additionalProperties: true
@@ -2199,6 +3164,150 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  ComponentMappingDefinition:
+    title: Component Mapping Definition
+    description: (This component is experimental. Use at your own risk.) Specifies a mapping definition to update or add fields in a record or configuration. This allows dynamic mapping of data by interpolating values into the template based on provided contexts.
+    type: object
+    required:
+      - type
+      - field_path
+      - value
+    properties:
+      type:
+        type: string
+        enum: [ComponentMappingDefinition]
+      field_path:
+        title: Field Path
+        description: A list of potentially nested fields indicating the full path where value will be added or updated.
+        type: array
+        items:
+          - type: string
+        interpolation_context:
+          - config
+          - components_values
+          - stream_slice
+          - stream_template_config
+        examples:
+          - ["data"]
+          - ["data", "records"]
+          - ["data", 1, "name"]
+          - ["data", "{{ components_values.name }}"]
+          - ["data", "*", "record"]
+          - ["*", "**", "name"]
+      value:
+        title: Value
+        description: The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.
+        type: string
+        interpolation_context:
+          - config
+          - stream_template_config
+          - components_values
+          - stream_slice
+        examples:
+          - "{{ components_values['updates'] }}"
+          - "{{ components_values['MetaData']['LastUpdatedTime'] }}"
+          - "{{ config['segment_id'] }}"
+          - "{{ stream_slice['parent_id'] }}"
+          - "{{ stream_slice['extra_fields']['name'] }}"
+      value_type:
+        title: Value Type
+        description: The expected data type of the value. If omitted, the type will be inferred from the value provided.
+        "$ref": "#/definitions/ValueType"
+      $parameters:
+        type: object
+        additionalProperties: true
+  HttpComponentsResolver:
+    type: object
+    description: (This component is experimental. Use at your own risk.) Component resolve and populates stream templates with components fetched via an HTTP retriever.
+    properties:
+      type:
+        type: string
+        enum: [HttpComponentsResolver]
+      retriever:
+        title: Retriever
+        description: Component used to coordinate how records are extracted across stream slices and request pages.
+        anyOf:
+          - "$ref": "#/definitions/AsyncRetriever"
+          - "$ref": "#/definitions/CustomRetriever"
+          - "$ref": "#/definitions/SimpleRetriever"
+      components_mapping:
+        type: array
+        items:
+          "$ref": "#/definitions/ComponentMappingDefinition"
+      $parameters:
+        type: object
+        additionalProperties: true
+    required:
+      - type
+      - retriever
+      - components_mapping
+  StreamConfig:
+    title: Stream Config
+    description: (This component is experimental. Use at your own risk.) Describes how to get streams config from the source config.
+    type: object
+    required:
+      - type
+      - configs_pointer
+    properties:
+      type:
+        type: string
+        enum: [StreamConfig]
+      configs_pointer:
+        title: Configs Pointer
+        description: A list of potentially nested fields indicating the full path in source config file where streams configs located.
+        type: array
+        items:
+          - type: string
+        interpolation_context:
+          - parameters
+        examples:
+          - ["data"]
+          - ["data", "streams"]
+          - ["data", "{{ parameters.name }}"]
+      $parameters:
+        type: object
+        additionalProperties: true
+  ConfigComponentsResolver:
+    type: object
+    description: (This component is experimental. Use at your own risk.) Resolves and populates stream templates with components fetched from the source config.
+    properties:
+      type:
+        type: string
+        enum: [ConfigComponentsResolver]
+      stream_config:
+        "$ref": "#/definitions/StreamConfig"
+      components_mapping:
+        type: array
+        items:
+          "$ref": "#/definitions/ComponentMappingDefinition"
+      $parameters:
+        type: object
+        additionalProperties: true
+    required:
+      - type
+      - stream_config
+      - components_mapping
+  DynamicDeclarativeStream:
+    type: object
+    description: (This component is experimental. Use at your own risk.) A component that described how will be created declarative streams based on stream template.
+    properties:
+      type:
+        type: string
+        enum: [DynamicDeclarativeStream]
+      stream_template:
+        title: Stream Template
+        description: Reference to the stream template.
+        "$ref": "#/definitions/DeclarativeStream"
+      components_resolver:
+        title: Components Resolver
+        description: Component resolve and populates stream templates with components values.
+        anyOf:
+          - "$ref": "#/definitions/HttpComponentsResolver"
+          - "$ref": "#/definitions/ConfigComponentsResolver"
+    required:
+      - type
+      - stream_template
+      - components_resolver
 interpolation:
   variables:
     - title: config
@@ -2228,20 +3337,20 @@ interpolation:
           x-ratelimit-limit: "600"
           x-ratelimit-remaining: "598"
           x-ratelimit-reset: "39"
-    - title: last_records
-      description: List of records extracted from the last response received from the API.
-      type: list
+    - title: last_record
+      description: Last record extracted from the response received from the API.
+      type: object
+      examples:
+        - name: "Test List: 19"
+          id: 0236d6d2
+          contact_count: 20
+          _metadata:
+            self: https://api.sendgrid.com/v3/marketing/lists/0236d6d2
+    - title: last_page_size
+      description: Number of records extracted from the last response received from the API.
+      type: object
       examples:
-        - - name: "Test List: 19"
-            id: 0236d6d2
-            contact_count: 20
-            _metadata:
-              self: https://api.sendgrid.com/v3/marketing/lists/0236d6d2
-          - name: List for CI tests, number 30
-            id: 041ee031
-            contact_count: 0
-            _metadata:
-              self: https://api.sendgrid.com/v3/marketing/lists/041ee031
+        - 2
     - title: next_page_token
       description: Object describing the token to fetch the next page of records. The object has a single key "next_page_token".
       type: object
@@ -2289,21 +3398,21 @@ interpolation:
         - created_at: "2020-01-01 00:00:00.000+00:00"
         - updated_at: "2020-01-02 00:00:00.000+00:00"
   macros:
-    - title: Now (UTC)
+    - title: now_utc
       description: Returns the current date and time in the UTC timezone.
       arguments: {}
       return_type: Datetime
       examples:
         - "'{{ now_utc() }}' -> '2021-09-01 00:00:00+00:00'"
         - "'{{ now_utc().strftime('%Y-%m-%d') }}' -> '2021-09-01'"
-    - title: Today (UTC)
+    - title: today_utc
       description: Returns the current date in UTC timezone. The output is a date object.
       arguments: {}
       return_type: Date
       examples:
         - "'{{ today_utc() }}' -> '2021-09-01'"
         - "'{{ today_utc().strftime('%Y/%m/%d')}}' -> '2021/09/01'"
-    - title: Timestamp
+    - title: timestamp
       description: Converts a number or a string representing a datetime (formatted as ISO8601) to a timestamp. If the input is a number, it is converted to an int. If no timezone is specified, the string is interpreted as UTC.
       arguments:
         datetime: A string formatted as ISO8601 or an integer representing a unix timestamp
@@ -2314,7 +3423,7 @@ interpolation:
         - "'{{ timestamp('2022-02-28T00:00:00Z') }}' -> 1646006400"
         - "'{{ timestamp('2022-02-28 00:00:00Z') }}' -> 1646006400"
         - "'{{ timestamp('2022-02-28T00:00:00-08:00') }}' -> 1646035200"
-    - title: Max
+    - title: max
       description: Returns the largest object of a iterable, or or two or more arguments.
       arguments:
         args: iterable or a sequence of two or more arguments
@@ -2322,7 +3431,7 @@ interpolation:
       examples:
         - "'{{ max(2, 3) }}' -> 3"
         - "'{{ max([2, 3]) }}' -> 3"
-    - title: Day Delta
+    - title: day_delta
       description: Returns the datetime of now() + num_days.
       arguments:
         num_days: The number of days to add to now
@@ -2332,8 +3441,8 @@ interpolation:
         - "'{{ day_delta(1) }}' -> '2021-09-02T00:00:00.000000+0000'"
         - "'{{ day_delta(-1) }}' -> '2021-08-31:00:00.000000+0000'"
         - "'{{ day_delta(25, format='%Y-%m-%d') }}' -> '2021-09-02'"
-    - title: Duration
-      description: Converts an ISO8601 duratioin to datetime.timedelta.
+    - title: duration
+      description: Converts an ISO8601 duration to datetime timedelta.
       arguments:
         duration_string: "A string representing an ISO8601 duration. See https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm for more details."
       return_type: datetime.timedelta
@@ -2341,17 +3450,19 @@ interpolation:
         - "'{{ duration('P1D') }}' -> '1 day, 0:00:00'"
         - "'{{ duration('P6DT23H') }}' -> '6 days, 23:00:00'"
         - "'{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}' -> '2021-08-31T00:00:00Z'"
-    - title: Format Datetime
+    - title: format_datetime
       description: Converts a datetime or a datetime-string to the specified format.
       arguments:
         datetime: The datetime object or a string to convert. If datetime is a string, it must be formatted as ISO8601.
-        format: The datetime format
+        format: The datetime format.
+        input_format: (optional) The datetime format in the case it is an string.
       return_type: str
       examples:
         - "{{ format_datetime(config['start_time'], '%Y-%m-%d') }}"
         - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ') }}"
+        - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ', '%a, %d %b %Y %H:%M:%S %z') }}"
   filters:
-    - title: Hash
+    - title: hash
       description: Convert the specified value to a hashed string.
       arguments:
         hash_type: Valid hash type for converts ('md5' as default value).
@@ -2361,15 +3472,29 @@ interpolation:
         - "{{ 'Test client_secret' | hash() }} -> '3032d57a12f76b61a820e47b9a5a0cbb'"
         - "{{ 'Test client_secret' | hash('md5') }} -> '3032d57a12f76b61a820e47b9a5a0cbb'"
         - "{{ 'Test client_secret' | hash('md5', salt='salt') }} -> '5011a0168579c2d94cbbe1c6ad14327c'"
-    - title: Base64 encoder
+    - title: base64encode
       description: Convert the specified value to a string in the base64 format.
       arguments: {}
       return_type: str
       examples:
         - "{{ 'Test client_secret' | base64encode }} -> 'VGVzdCBjbGllbnRfc2VjcmV0'"
-    - title: Base64 decoder
+    - title: base64decode
       description: Decodes the specified base64 format value into a common string.
       arguments: {}
       return_type: str
       examples:
         - "{{ 'ZmFrZSByZWZyZXNoX3Rva2VuIHZhbHVl' | base64decode }} -> 'fake refresh_token value'"
+    - title: string
+      description: Converts the specified value to a string.
+      arguments: {}
+      return_type: str
+      examples:
+        - '{{ 1 | string }} -> "1"'
+        - '{{ ["hello", "world" | string }} -> "["hello", "world"]"'
+    - title: regex_search
+      description: Match the input string against a regular expression and return the first match.
+      arguments:
+        regex: The regular expression to search for. It must include a capture group.
+      return_type: str
+      examples:
+        - '{{ "goodbye, cruel world" | regex_search("goodbye,\s(.*)$") }} -> "cruel world"'