airbyte-cdk 6.20.1__py3-none-any.whl → 6.20.2.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/auth/oauth.py

@@ -39,7 +39,6 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
         token_expiry_date_format str: format of the datetime; provide it if expires_in is returned in datetime instead of seconds
         token_expiry_is_time_of_expiration bool: set True it if expires_in is returned as time of expiration instead of the number seconds until expiration
         refresh_request_body (Optional[Mapping[str, Any]]): The request body to send in the refresh request
-        refresh_request_headers (Optional[Mapping[str, Any]]): The request headers to send in the refresh request
         grant_type: The grant_type to request for access_token. If set to refresh_token, the refresh_token parameter has to be provided
         message_repository (MessageRepository): the message repository used to emit logs on HTTP requests
     """
@@ -57,13 +56,8 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
     token_expiry_is_time_of_expiration: bool = False
     access_token_name: Union[InterpolatedString, str] = "access_token"
     access_token_value: Optional[Union[InterpolatedString, str]] = None
-    client_id_name: Union[InterpolatedString, str] = "client_id"
-    client_secret_name: Union[InterpolatedString, str] = "client_secret"
     expires_in_name: Union[InterpolatedString, str] = "expires_in"
-    refresh_token_name: Union[InterpolatedString, str] = "refresh_token"
     refresh_request_body: Optional[Mapping[str, Any]] = None
-    refresh_request_headers: Optional[Mapping[str, Any]] = None
-    grant_type_name: Union[InterpolatedString, str] = "grant_type"
     grant_type: Union[InterpolatedString, str] = "refresh_token"
     message_repository: MessageRepository = NoopMessageRepository()
 
@@ -75,15 +69,8 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
             )
         else:
             self._token_refresh_endpoint = None
-        self._client_id_name = InterpolatedString.create(self.client_id_name, parameters=parameters)
         self._client_id = InterpolatedString.create(self.client_id, parameters=parameters)
-        self._client_secret_name = InterpolatedString.create(
-            self.client_secret_name, parameters=parameters
-        )
         self._client_secret = InterpolatedString.create(self.client_secret, parameters=parameters)
-        self._refresh_token_name = InterpolatedString.create(
-            self.refresh_token_name, parameters=parameters
-        )
         if self.refresh_token is not None:
             self._refresh_token: Optional[InterpolatedString] = InterpolatedString.create(
                 self.refresh_token, parameters=parameters
@@ -96,16 +83,10 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
         self.expires_in_name = InterpolatedString.create(
             self.expires_in_name, parameters=parameters
         )
-        self.grant_type_name = InterpolatedString.create(
-            self.grant_type_name, parameters=parameters
-        )
         self.grant_type = InterpolatedString.create(self.grant_type, parameters=parameters)
         self._refresh_request_body = InterpolatedMapping(
             self.refresh_request_body or {}, parameters=parameters
         )
-        self._refresh_request_headers = InterpolatedMapping(
-            self.refresh_request_headers or {}, parameters=parameters
-        )
         self._token_expiry_date: pendulum.DateTime = (
             pendulum.parse(
                 InterpolatedString.create(self.token_expiry_date, parameters=parameters).eval(
@@ -141,27 +122,18 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
             return refresh_token_endpoint
         return None
 
-    def get_client_id_name(self) -> str:
-        return self._client_id_name.eval(self.config)  # type: ignore # eval returns a string in this context
-
     def get_client_id(self) -> str:
         client_id: str = self._client_id.eval(self.config)
         if not client_id:
             raise ValueError("OAuthAuthenticator was unable to evaluate client_id parameter")
         return client_id
 
-    def get_client_secret_name(self) -> str:
-        return self._client_secret_name.eval(self.config)  # type: ignore # eval returns a string in this context
-
     def get_client_secret(self) -> str:
         client_secret: str = self._client_secret.eval(self.config)
         if not client_secret:
             raise ValueError("OAuthAuthenticator was unable to evaluate client_secret parameter")
         return client_secret
 
-    def get_refresh_token_name(self) -> str:
-        return self._refresh_token_name.eval(self.config)  # type: ignore # eval returns a string in this context
-
     def get_refresh_token(self) -> Optional[str]:
         return None if self._refresh_token is None else str(self._refresh_token.eval(self.config))
 
@@ -174,18 +146,12 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
     def get_expires_in_name(self) -> str:
         return self.expires_in_name.eval(self.config)  # type: ignore # eval returns a string in this context
 
-    def get_grant_type_name(self) -> str:
-        return self.grant_type_name.eval(self.config)  # type: ignore # eval returns a string in this context
-
     def get_grant_type(self) -> str:
         return self.grant_type.eval(self.config)  # type: ignore # eval returns a string in this context
 
     def get_refresh_request_body(self) -> Mapping[str, Any]:
         return self._refresh_request_body.eval(self.config)
 
-    def get_refresh_request_headers(self) -> Mapping[str, Any]:
-        return self._refresh_request_headers.eval(self.config)
-
     def get_token_expiry_date(self) -> pendulum.DateTime:
         return self._token_expiry_date  # type: ignore # _token_expiry_date is a pendulum.DateTime. It is never None despite what mypy thinks
 

airbyte_cdk/sources/declarative/checks/__init__.py

@@ -1,24 +1,8 @@
 #
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from typing import Mapping
-
-from pydantic.v1 import BaseModel
-
-from airbyte_cdk.sources.declarative.checks.check_dynamic_stream import CheckDynamicStream
 from airbyte_cdk.sources.declarative.checks.check_stream import CheckStream
 from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
-from airbyte_cdk.sources.declarative.models import (
-    CheckDynamicStream as CheckDynamicStreamModel,
-)
-from airbyte_cdk.sources.declarative.models import (
-    CheckStream as CheckStreamModel,
-)
-
-COMPONENTS_CHECKER_TYPE_MAPPING: Mapping[str, type[BaseModel]] = {
-    "CheckStream": CheckStreamModel,
-    "CheckDynamicStream": CheckDynamicStreamModel,
-}
 
-__all__ = ["CheckStream", "CheckDynamicStream", "ConnectionChecker"]
+__all__ = ["CheckStream", "ConnectionChecker"]

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -20,6 +20,9 @@ from airbyte_cdk.sources.declarative.extractors.record_filter import (
     ClientSideIncrementalRecordFilterDecorator,
 )
 from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
+from airbyte_cdk.sources.declarative.incremental.per_partition_with_global import (
+    PerPartitionWithGlobalCursor,
+)
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
@@ -32,7 +35,7 @@ from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
     ModelToComponentFactory,
 )
 from airbyte_cdk.sources.declarative.requesters import HttpRequester
-from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever
+from airbyte_cdk.sources.declarative.retrievers import Retriever, SimpleRetriever
 from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import (
     DeclarativePartitionFactory,
     StreamSlicerPartitionGenerator,
@@ -230,21 +233,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                         stream_state=stream_state,
                     )
 
-                    retriever = declarative_stream.retriever
-
-                    # This is an optimization so that we don't invoke any cursor or state management flows within the
-                    # low-code framework because state management is handled through the ConcurrentCursor.
-                    if declarative_stream and isinstance(retriever, SimpleRetriever):
-                        # Also a temporary hack. In the legacy Stream implementation, as part of the read,
-                        # set_initial_state() is called to instantiate incoming state on the cursor. Although we no
-                        # longer rely on the legacy low-code cursor for concurrent checkpointing, low-code components
-                        # like StopConditionPaginationStrategyDecorator and ClientSideIncrementalRecordFilterDecorator
-                        # still rely on a DatetimeBasedCursor that is properly initialized with state.
-                        if retriever.cursor:
-                            retriever.cursor.set_initial_state(stream_state=stream_state)
-                        # We zero it out here, but since this is a cursor reference, the state is still properly
-                        # instantiated for the other components that reference it
-                        retriever.cursor = None
+                    retriever = self._get_retriever(declarative_stream, stream_state)
 
                     partition_generator = StreamSlicerPartitionGenerator(
                         DeclarativePartitionFactory(
@@ -304,6 +293,60 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             cursor=final_state_cursor,
                         )
                     )
+                elif (
+                    incremental_sync_component_definition
+                    and incremental_sync_component_definition.get("type", "")
+                    == DatetimeBasedCursorModel.__name__
+                    and self._stream_supports_concurrent_partition_processing(
+                        declarative_stream=declarative_stream
+                    )
+                    and hasattr(declarative_stream.retriever, "stream_slicer")
+                    and isinstance(
+                        declarative_stream.retriever.stream_slicer, PerPartitionWithGlobalCursor
+                    )
+                ):
+                    stream_state = state_manager.get_stream_state(
+                        stream_name=declarative_stream.name, namespace=declarative_stream.namespace
+                    )
+                    partition_router = declarative_stream.retriever.stream_slicer._partition_router
+
+                    perpartition_cursor = (
+                        self._constructor.create_concurrent_cursor_from_perpartition_cursor(
+                            state_manager=state_manager,
+                            model_type=DatetimeBasedCursorModel,
+                            component_definition=incremental_sync_component_definition,
+                            stream_name=declarative_stream.name,
+                            stream_namespace=declarative_stream.namespace,
+                            config=config or {},
+                            stream_state=stream_state,
+                            partition_router=partition_router,
+                        )
+                    )
+
+                    retriever = self._get_retriever(declarative_stream, stream_state)
+
+                    partition_generator = StreamSlicerPartitionGenerator(
+                        DeclarativePartitionFactory(
+                            declarative_stream.name,
+                            declarative_stream.get_json_schema(),
+                            retriever,
+                            self.message_repository,
+                        ),
+                        perpartition_cursor,
+                    )
+
+                    concurrent_streams.append(
+                        DefaultStream(
+                            partition_generator=partition_generator,
+                            name=declarative_stream.name,
+                            json_schema=declarative_stream.get_json_schema(),
+                            availability_strategy=AlwaysAvailableAvailabilityStrategy(),
+                            primary_key=get_primary_key_from_stream(declarative_stream.primary_key),
+                            cursor_field=perpartition_cursor.cursor_field.cursor_field_key,
+                            logger=self.logger,
+                            cursor=perpartition_cursor,
+                        )
+                    )
                 else:
                     synchronous_streams.append(declarative_stream)
             else:
@@ -394,6 +437,27 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             return False
         return True
 
+    def _get_retriever(
+        self, declarative_stream: DeclarativeStream, stream_state: Mapping[str, Any]
+    ) -> Retriever:
+        retriever = declarative_stream.retriever
+
+        # This is an optimization so that we don't invoke any cursor or state management flows within the
+        # low-code framework because state management is handled through the ConcurrentCursor.
+        if declarative_stream and isinstance(retriever, SimpleRetriever):
+            # Also a temporary hack. In the legacy Stream implementation, as part of the read,
+            # set_initial_state() is called to instantiate incoming state on the cursor. Although we no
+            # longer rely on the legacy low-code cursor for concurrent checkpointing, low-code components
+            # like StopConditionPaginationStrategyDecorator and ClientSideIncrementalRecordFilterDecorator
+            # still rely on a DatetimeBasedCursor that is properly initialized with state.
+            if retriever.cursor:
+                retriever.cursor.set_initial_state(stream_state=stream_state)
+            # We zero it out here, but since this is a cursor reference, the state is still properly
+            # instantiated for the other components that reference it
+            retriever.cursor = None
+
+        return retriever
+
     @staticmethod
     def _select_streams(
         streams: List[AbstractStream], configured_catalog: ConfiguredAirbyteCatalog

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -18,9 +18,7 @@ properties:
     type: string
     enum: [DeclarativeSource]
   check:
-    anyOf:
-      - "$ref": "#/definitions/CheckStream"
-      - "$ref": "#/definitions/CheckDynamicStream"
+    "$ref": "#/definitions/CheckStream"
   streams:
     type: array
     items:
@@ -305,21 +303,6 @@ definitions:
         examples:
           - ["users"]
           - ["users", "contacts"]
-  CheckDynamicStream:
-    title: Dynamic Streams to Check
-    description: (This component is experimental. Use at your own risk.) Defines the dynamic streams to try reading when running a check operation.
-    type: object
-    required:
-      - type
-      - stream_count
-    properties:
-      type:
-        type: string
-        enum: [CheckDynamicStream]
-      stream_count:
-        title: Stream Count
-        description: Numbers of the streams to try reading from when running a check operation.
-        type: integer
   CompositeErrorHandler:
     title: Composite Error Handler
     description: Error handler that sequentially iterates over a list of error handlers.
@@ -695,7 +678,7 @@ definitions:
     properties:
       type:
         type: string
-        enum: [CustomSchemaNormalization]
+        enum: [ CustomSchemaNormalization ]
       class_name:
         title: Class Name
         description: Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.
@@ -1064,13 +1047,6 @@ definitions:
       type:
         type: string
         enum: [OAuthAuthenticator]
-      client_id_name:
-        title: Client ID Property Name
-        description: The name of the property to use to refresh the `access_token`.
-        type: string
-        default: "client_id"
-        examples:
-          - custom_app_id
       client_id:
         title: Client ID
         description: The OAuth client ID. Fill it in the user inputs.
@@ -1078,13 +1054,6 @@ definitions:
         examples:
           - "{{ config['client_id }}"
           - "{{ config['credentials']['client_id }}"
-      client_secret_name:
-        title: Client Secret Property Name
-        description: The name of the property to use to refresh the `access_token`.
-        type: string
-        default: "client_secret"
-        examples:
-          - custom_app_secret
       client_secret:
         title: Client Secret
         description: The OAuth client secret. Fill it in the user inputs.
@@ -1092,13 +1061,6 @@ definitions:
         examples:
           - "{{ config['client_secret }}"
           - "{{ config['credentials']['client_secret }}"
-      refresh_token_name:
-        title: Refresh Token Property Name
-        description: The name of the property to use to refresh the `access_token`.
-        type: string
-        default: "refresh_token"
-        examples:
-          - custom_app_refresh_value
       refresh_token:
         title: Refresh Token
         description: Credential artifact used to get a new access token.
@@ -1132,13 +1094,6 @@ definitions:
         default: "expires_in"
         examples:
           - expires_in
-      grant_type_name:
-        title: Grant Type Property Name
-        description: The name of the property to use to refresh the `access_token`.
-        type: string
-        default: "grant_type"
-        examples:
-          - custom_grant_type
       grant_type:
         title: Grant Type
         description: Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.
@@ -1156,14 +1111,6 @@ definitions:
           - applicationId: "{{ config['application_id'] }}"
             applicationSecret: "{{ config['application_secret'] }}"
             token: "{{ config['token'] }}"
-      refresh_request_headers:
-        title: Refresh Request Headers
-        description: Headers of the request sent to get a new access token.
-        type: object
-        additionalProperties: true
-        examples:
-          - Authorization: "<AUTH_TOKEN>"
-            Content-Type: "application/x-www-form-urlencoded"
       scopes:
         title: Scopes
         description: List of scopes that should be granted to the access token.
@@ -1788,10 +1735,6 @@ definitions:
           - type: array
             items:
               type: string
-      condition:
-        type: string
-        interpolation_context:
-          - raw_schema
   SchemaTypeIdentifier:
     title: Schema Type Identifier
     description: (This component is experimental. Use at your own risk.) Identifies schema details for dynamic schema extraction and processing.
@@ -2261,15 +2204,15 @@ definitions:
           Pertains to the fields defined by the connector relating to the OAuth flow.
 
           Interpolation capabilities:
-          - The variables placeholders are declared as `{{my_var}}`.
-          - The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.
+          - The variables placeholders are declared as `{my_var}`.
+          - The nested resolution variables like `{{my_nested_var}}` is allowed as well.
 
           - The allowed interpolation context is:
-            + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}
-            + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}
-            + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}
-            + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}
-            + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}
+            + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}
+            + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}
+            + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}
+            + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}
+            + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}
 
           Examples:
             - The TikTok Marketing DeclarativeOAuth spec:
@@ -2278,12 +2221,12 @@ definitions:
                 "type": "object",
                 "additionalProperties": false,
                 "properties": {
-                    "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",
+                    "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
                     "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",
                     "access_token_params": {
-                        "{{ auth_code_key }}": "{{ auth_code_value }}",
-                        "{{ client_id_key }}": "{{ client_id_value }}",
-                        "{{ client_secret_key }}": "{{ client_secret_value }}"
+                        "{auth_code_key}": "{{auth_code_key}}",
+                        "{client_id_key}": "{{client_id_key}}",
+                        "{client_secret_key}": "{{client_secret_key}}"
                     },
                     "access_token_headers": {
                         "Content-Type": "application/json",
@@ -2301,6 +2244,7 @@ definitions:
         required:
           - consent_url
           - access_token_url
+          - extract_output
         properties:
           consent_url:
             title: Consent URL
@@ -2309,8 +2253,8 @@ definitions:
               The DeclarativeOAuth Specific string URL string template to initiate the authentication.
               The placeholders are replaced during the processing to provide neccessary values.
             examples:
-              - https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}
-              - https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}
+              - https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}
+              - https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}
           scope:
             title: Scopes
             type: string
@@ -2325,7 +2269,7 @@ definitions:
               The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.
               The placeholders are replaced during the processing to provide neccessary values.
             examples:
-              - https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}
+              - https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}
           access_token_headers:
             title: Access Token Headers
             type: object
@@ -2334,7 +2278,7 @@ definitions:
               The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
             examples:
               - {
-                  "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}",
+                  "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}",
                 }
           access_token_params:
             title: Access Token Query Params (Json Encoded)
@@ -2345,9 +2289,9 @@ definitions:
               When this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.
             examples:
               - {
-                  "{{ auth_code_key }}": "{{ auth_code_value }}",
-                  "{{ client_id_key }}": "{{ client_id_value }}",
-                  "{{ client_secret_key }}": "{{ client_secret_value }}",
+                  "{auth_code_key}": "{{auth_code_key}}",
+                  "{client_id_key}": "{{client_id_key}}",
+                  "{client_secret_key}": "{{client_secret_key}}",
                 }
           extract_output:
             title: Extract Output
@@ -2915,7 +2859,6 @@ definitions:
       parser:
         anyOf:
           - "$ref": "#/definitions/GzipParser"
-          - "$ref": "#/definitions/JsonParser"
           - "$ref": "#/definitions/JsonLineParser"
           - "$ref": "#/definitions/CsvParser"
   #    PARSERS
@@ -2932,20 +2875,6 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/JsonLineParser"
           - "$ref": "#/definitions/CsvParser"
-          - "$ref": "#/definitions/JsonParser"
-  JsonParser:
-    title: JsonParser
-    description: Parser used for parsing str, bytes, or bytearray data and returning data in a dictionary format.
-    type: object
-    required:
-      - type
-    properties:
-      type:
-        type: string
-        enum: [JsonParser]
-      encoding:
-        type: string
-        default: utf-8
   JsonLineParser:
     type: object
     required:
@@ -3048,11 +2977,6 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/CustomRequester"
           - "$ref": "#/definitions/HttpRequester"
-      url_requester:
-        description: Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.
-        anyOf:
-          - "$ref": "#/definitions/CustomRequester"
-          - "$ref": "#/definitions/HttpRequester"
       download_requester:
         description: Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.
         anyOf:

airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -7,12 +7,9 @@ from dataclasses import dataclass
 from io import BufferedIOBase, TextIOWrapper
 from typing import Any, Generator, MutableMapping, Optional
 
-import orjson
 import requests
 
-from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
-from airbyte_cdk.utils import AirbyteTracedException
 
 logger = logging.getLogger("airbyte")
 
@@ -45,46 +42,6 @@ class GzipParser(Parser):
             yield from self.inner_parser.parse(gzipobj)
 
 
-@dataclass
-class JsonParser(Parser):
-    encoding: str = "utf-8"
-
-    def parse(self, data: BufferedIOBase) -> Generator[MutableMapping[str, Any], None, None]:
-        """
-        Attempts to deserialize data using orjson library. As an extra layer of safety we fallback on the json library to deserialize the data.
-        """
-        raw_data = data.read()
-        body_json = self._parse_orjson(raw_data) or self._parse_json(raw_data)
-
-        if body_json is None:
-            raise AirbyteTracedException(
-                message="Response JSON data failed to be parsed. See logs for more information.",
-                internal_message=f"Response JSON data failed to be parsed.",
-                failure_type=FailureType.system_error,
-            )
-
-        if isinstance(body_json, list):
-            yield from body_json
-        else:
-            yield from [body_json]
-
-    def _parse_orjson(self, raw_data: bytes) -> Optional[Any]:
-        try:
-            return orjson.loads(raw_data.decode(self.encoding))
-        except Exception as exc:
-            logger.debug(
-                f"Failed to parse JSON data using orjson library. Falling back to json library. {exc}"
-            )
-            return None
-
-    def _parse_json(self, raw_data: bytes) -> Optional[Any]:
-        try:
-            return json.loads(raw_data.decode(self.encoding))
-        except Exception as exc:
-            logger.error(f"Failed to parse JSON data using json library. {exc}")
-            return None
-
-
 @dataclass
 class JsonLineParser(Parser):
     encoding: Optional[str] = "utf-8"

airbyte_cdk/sources/declarative/extractors/record_filter.py

@@ -59,13 +59,11 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
 
     def __init__(
         self,
-        date_time_based_cursor: DatetimeBasedCursor,
-        substream_cursor: Optional[Union[PerPartitionWithGlobalCursor, GlobalSubstreamCursor]],
+        cursor: Union[DatetimeBasedCursor, PerPartitionWithGlobalCursor, GlobalSubstreamCursor],
         **kwargs: Any,
     ):
         super().__init__(**kwargs)
-        self._date_time_based_cursor = date_time_based_cursor
-        self._substream_cursor = substream_cursor
+        self._cursor = cursor
 
     def filter_records(
         self,
@@ -77,7 +75,7 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
         records = (
             record
             for record in records
-            if (self._substream_cursor or self._date_time_based_cursor).should_be_synced(
+            if self._cursor.should_be_synced(
                 # Record is created on the fly to align with cursors interface; stream name is ignored as we don't need it here
                 # Record stream name is empty cause it is not used durig the filtering
                 Record(data=record, associated_slice=stream_slice, stream_name="")

airbyte_cdk/sources/declarative/incremental/__init__.py

@@ -2,6 +2,10 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+from airbyte_cdk.sources.declarative.incremental.concurrent_partition_cursor import (
+    ConcurrentCursorFactory,
+    ConcurrentPerPartitionCursor,
+)
 from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
 from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
 from airbyte_cdk.sources.declarative.incremental.global_substream_cursor import (
@@ -21,6 +25,8 @@ from airbyte_cdk.sources.declarative.incremental.resumable_full_refresh_cursor i
 
 __all__ = [
     "CursorFactory",
+    "ConcurrentCursorFactory",
+    "ConcurrentPerPartitionCursor",
     "DatetimeBasedCursor",
     "DeclarativeCursor",
     "GlobalSubstreamCursor",