airbyte-cdk 6.7.0rc3__py3-none-any.whl → 6.7.1.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1972,6 +1972,173 @@ definitions:
           - app_id:
               type: string
               path_in_connector_config: ["info", "app_id"]
+      oauth_connector_input_specification:
+        title: DeclarativeOAuth Connector Specification
+        description: |-
+          The DeclarativeOAuth specific blob.
+          Pertains to the fields defined by the connector relating to the OAuth flow.
+
+          Interpolation capabilities:
+          - The variables placeholders are declared as `{my_var}`.
+          - The nested resolution variables like `{{my_nested_var}}` is allowed as well.
+
+          - The allowed interpolation context is:
+            + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}
+            + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}
+            + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}
+            + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}
+            + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}
+
+          Examples:
+            - The TikTok Marketing DeclarativeOAuth spec:
+            {
+              "oauth_connector_input_specification": {
+                "type": "object",
+                "additionalProperties": false,
+                "properties": {
+                    "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
+                    "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",
+                    "access_token_params": {
+                        "{auth_code_key}": "{{auth_code_key}}",
+                        "{client_id_key}": "{{client_id_key}}",
+                        "{client_secret_key}": "{{client_secret_key}}"
+                    },
+                    "access_token_headers": {
+                        "Content-Type": "application/json",
+                        "Accept": "application/json"
+                    },
+                    "extract_output": ["data.access_token"],
+                    "client_id_key": "app_id",
+                    "client_secret_key": "secret",
+                    "auth_code_key": "auth_code"
+                }
+              }
+            }
+        type: object
+        additionalProperties: true
+        required:
+          - consent_url
+          - access_token_url
+          - extract_output
+        properties:
+          consent_url:
+            title: DeclarativeOAuth Consent URL
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific string URL string template to initiate the authentication.
+              The placeholders are replaced during the processing to provide neccessary values.
+            examples:
+              - consent_url: https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}
+              - consent_url: https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}
+          scope:
+            title: (Optional) DeclarativeOAuth Scope
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.
+            examples:
+              - scope: user:read user:read_orders workspaces:read
+          access_token_url:
+            title: DeclarativeOAuth Access Token URL
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.
+              The placeholders are replaced during the processing to provide neccessary values.
+            examples:
+              - access_token_url: https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}} 
+          access_token_headers:
+            title: (Optional) DeclarativeOAuth Access Token Headers
+            type: object
+            additionalProperties: true
+            description: |-
+              The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
+            examples:
+              - access_token_headers: {
+                  "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
+                }
+          access_token_params:
+            title: (Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)
+            type: object
+            additionalProperties: true
+            description: |-
+              The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
+              When this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.
+            examples:
+              - access_token_params: {
+                  "{auth_code_key}": "{{auth_code_key}}",
+                  "{client_id_key}": "{{client_id_key}}",
+                  "{client_secret_key}": "{{client_secret_key}}"
+                }
+          extract_output:
+            title: DeclarativeOAuth Extract Output
+            type: array
+            items:
+              type: string
+            description: |-
+              The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.                
+            examples:
+              - extract_output: ["access_token", "refresh_token", "other_field"]
+          state:
+            title: (Optional) DeclarativeOAuth Configurable State Query Param
+            type: object
+            additionalProperties: true
+            required:
+              - min
+              - max
+            description: |-
+              The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,
+              including length and complexity.                
+            properties:
+              min:
+                type: integer
+              max:
+                type: integer
+            examples:
+              - state: {
+                  "min": 7,
+                  "max": 128,
+                }
+          client_id_key:
+            title: (Optional) DeclarativeOAuth Client ID Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.
+            examples:
+              - client_id_key: "my_custom_client_id_key_name"
+          client_secret_key:
+            title: (Optional) DeclarativeOAuth Client Secret Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.
+            examples:
+              - client_secret_key: "my_custom_client_secret_key_name"
+          scope_key:
+            title: (Optional) DeclarativeOAuth Scope Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.
+            examples:
+              - scope_key: "my_custom_scope_key_key_name"
+          state_key:
+            title: (Optional) DeclarativeOAuth State Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.                
+            examples:
+              - state_key: "my_custom_state_key_key_name"
+          auth_code_key:
+            title: (Optional) DeclarativeOAuth Auth Code Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.                
+            examples:
+              - auth_code_key: "my_custom_auth_code_key_name"
+          redirect_uri_key:
+            title: (Optional) DeclarativeOAuth Redirect URI Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.
+            examples:
+              - redirect_uri_key: "my_custom_redirect_uri_key_name"
       complete_oauth_output_specification:
         title: "OAuth output specification"
         description: |-

airbyte_cdk/sources/declarative/decoders/json_decoder.py

@@ -5,11 +5,12 @@ import codecs
 import logging
 from dataclasses import InitVar, dataclass
 from gzip import decompress
-from typing import Any, Generator, Mapping, MutableMapping, List, Optional
+from typing import Any, Generator, List, Mapping, MutableMapping, Optional
 
+import orjson
 import requests
+
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
-import orjson
 
 logger = logging.getLogger("airbyte")
 

airbyte_cdk/sources/declarative/decoders/noop_decoder.py

@@ -4,6 +4,7 @@ import logging
 from typing import Any, Generator, Mapping
 
 import requests
+
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
 
 logger = logging.getLogger("airbyte")

airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py

@@ -7,6 +7,7 @@ from dataclasses import dataclass
 from typing import Any, Generator, MutableMapping
 
 import requests
+
 from airbyte_cdk.sources.declarative.decoders import Decoder
 
 logger = logging.getLogger("airbyte")

airbyte_cdk/sources/declarative/decoders/xml_decoder.py

@@ -9,6 +9,7 @@ from xml.parsers.expat import ExpatError
 
 import requests
 import xmltodict
+
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
 
 logger = logging.getLogger("airbyte")

airbyte_cdk/sources/declarative/extractors/dpath_extractor.py

@@ -7,6 +7,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Union
 
 import dpath
 import requests
+
 from airbyte_cdk.sources.declarative.decoders import Decoder, JsonDecoder
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString

airbyte_cdk/sources/declarative/extractors/http_selector.py

@@ -6,6 +6,7 @@ from abc import abstractmethod
 from typing import Any, Iterable, Mapping, Optional
 
 import requests
+
 from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
 
 

airbyte_cdk/sources/declarative/extractors/record_filter.py

@@ -1,7 +1,6 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-import datetime
 from dataclasses import InitVar, dataclass
 from typing import Any, Iterable, Mapping, Optional, Union
 
@@ -11,7 +10,7 @@ from airbyte_cdk.sources.declarative.incremental import (
     PerPartitionWithGlobalCursor,
 )
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
-from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
 
 
 @dataclass
@@ -68,20 +67,6 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
         self._date_time_based_cursor = date_time_based_cursor
         self._substream_cursor = substream_cursor
 
-    @property
-    def _cursor_field(self) -> str:
-        return self._date_time_based_cursor.cursor_field.eval(self._date_time_based_cursor.config)  # type: ignore # eval returns a string in this context
-
-    @property
-    def _start_date_from_config(self) -> datetime.datetime:
-        return self._date_time_based_cursor._start_datetime.get_datetime(
-            self._date_time_based_cursor.config
-        )
-
-    @property
-    def _end_datetime(self) -> datetime.datetime:
-        return self._date_time_based_cursor.select_best_end_datetime()
-
     def filter_records(
         self,
         records: Iterable[Mapping[str, Any]],
@@ -89,16 +74,14 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[Mapping[str, Any]]:
-        state_value = self._get_state_value(
-            stream_state, stream_slice or StreamSlice(partition={}, cursor_slice={})
-        )
-        filter_date: datetime.datetime = self._get_filter_date(state_value)
         records = (
             record
             for record in records
-            if self._end_datetime
-            >= self._date_time_based_cursor.parse_date(record[self._cursor_field])
-            >= filter_date
+            if (self._substream_cursor or self._date_time_based_cursor).should_be_synced(
+                # Record is created on the fly to align with cursors interface; stream name is ignored as we don't need it here
+                # Record stream name is empty cause it is not used durig the filtering
+                Record(data=record, associated_slice=stream_slice, stream_name="")
+            )
         )
         if self.condition:
             records = super().filter_records(
@@ -108,28 +91,3 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
                 next_page_token=next_page_token,
             )
         yield from records
-
-    def _get_state_value(
-        self, stream_state: StreamState, stream_slice: StreamSlice
-    ) -> Optional[str]:
-        """
-        Return cursor_value or None in case it was not found.
-        Cursor_value may be empty if:
-            1. It is an initial sync => no stream_state exist at all.
-            2. In Parent-child stream, and we already make initial sync, so stream_state is present.
-               During the second read, we receive one extra record from parent and therefore no stream_state for this record will be found.
-
-        :param StreamState stream_state: State
-        :param StreamSlice stream_slice: Current Stream slice
-        :return Optional[str]: cursor_value in case it was found, otherwise None.
-        """
-        state = (self._substream_cursor or self._date_time_based_cursor).select_state(stream_slice)
-
-        return state.get(self._cursor_field) if state else None
-
-    def _get_filter_date(self, state_value: Optional[str]) -> datetime.datetime:
-        start_date_parsed = self._start_date_from_config
-        if state_value:
-            return max(start_date_parsed, self._date_time_based_cursor.parse_date(state_value))
-        else:
-            return start_date_parsed

airbyte_cdk/sources/declarative/extractors/record_selector.py

@@ -3,12 +3,14 @@
 #
 
 from dataclasses import InitVar, dataclass, field
-from typing import Any, Iterable, List, Mapping, Optional
+from typing import Any, Iterable, List, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.models import SchemaNormalization
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
@@ -37,11 +39,34 @@ class RecordSelector(HttpSelector):
     config: Config
     parameters: InitVar[Mapping[str, Any]]
     schema_normalization: TypeTransformer
+    name: str
+    _name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
     record_filter: Optional[RecordFilter] = None
     transformations: List[RecordTransformation] = field(default_factory=lambda: [])
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters
+        self._name = (
+            InterpolatedString(self._name, parameters=parameters)
+            if isinstance(self._name, str)
+            else self._name
+        )
+
+    @property  # type: ignore
+    def name(self) -> str:
+        """
+        :return: Stream name
+        """
+        return (
+            str(self._name.eval(self.config))
+            if isinstance(self._name, InterpolatedString)
+            else self._name
+        )
+
+    @name.setter
+    def name(self, value: str) -> None:
+        if not isinstance(value, property):
+            self._name = value
 
     def select_records(
         self,
@@ -85,7 +110,7 @@ class RecordSelector(HttpSelector):
         transformed_data = self._transform(filtered_data, stream_state, stream_slice)
         normalized_data = self._normalize_by_schema(transformed_data, schema=records_schema)
         for data in normalized_data:
-            yield Record(data, stream_slice)
+            yield Record(data=data, stream_name=self.name, associated_slice=stream_slice)
 
     def _normalize_by_schema(
         self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]
@@ -125,6 +150,9 @@ class RecordSelector(HttpSelector):
         for record in records:
             for transformation in self.transformations:
                 transformation.transform(
-                    record, config=self.config, stream_state=stream_state, stream_slice=stream_slice
-                )  # type: ignore  # record has type Mapping[str, Any], but Dict[str, Any] expected
+                    record,  # type: ignore  # record has type Mapping[str, Any], but Dict[str, Any] expected
+                    config=self.config,
+                    stream_state=stream_state,
+                    stream_slice=stream_slice,
+                )
             yield record

airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py

@@ -10,9 +10,10 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
 
 import pandas as pd
 import requests
-from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from numpy import nan
 
+from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
+
 EMPTY_STR: str = ""
 DEFAULT_ENCODING: str = "utf-8"
 DOWNLOAD_CHUNK_SIZE: int = 1024 * 10

airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py

@@ -7,6 +7,8 @@ from dataclasses import InitVar, dataclass, field
 from datetime import timedelta
 from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
 
+from isodate import Duration, duration_isoformat, parse_duration
+
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
 from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
 from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
@@ -19,7 +21,6 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
 )
 from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
-from isodate import Duration, duration_isoformat, parse_duration
 
 
 @dataclass

airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py

@@ -340,8 +340,11 @@ class GlobalSubstreamCursor(DeclarativeCursor):
     @staticmethod
     def _convert_record_to_cursor_record(record: Record) -> Record:
         return Record(
-            record.data,
-            StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
+            data=record.data,
+            stream_name=record.stream_name,
+            associated_slice=StreamSlice(
+                partition={}, cursor_slice=record.associated_slice.cursor_slice
+            )
             if record.associated_slice
             else None,
         )

airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py

@@ -325,8 +325,11 @@ class PerPartitionCursor(DeclarativeCursor):
     @staticmethod
     def _convert_record_to_cursor_record(record: Record) -> Record:
         return Record(
-            record.data,
-            StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
+            data=record.data,
+            stream_name=record.stream_name,
+            associated_slice=StreamSlice(
+                partition={}, cursor_slice=record.associated_slice.cursor_slice
+            )
             if record.associated_slice
             else None,
         )

airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py

@@ -194,9 +194,7 @@ class PerPartitionWithGlobalCursor(DeclarativeCursor):
         )
 
     def should_be_synced(self, record: Record) -> bool:
-        return self._global_cursor.should_be_synced(
-            record
-        ) or self._per_partition_cursor.should_be_synced(record)
+        return self._get_active_cursor().should_be_synced(record)
 
     def is_greater_than_or_equal(self, first: Record, second: Record) -> bool:
         return self._global_cursor.is_greater_than_or_equal(first, second)

airbyte_cdk/sources/declarative/interpolation/jinja.py

@@ -6,15 +6,16 @@ import ast
 from functools import cache
 from typing import Any, Mapping, Optional, Tuple, Type
 
-from airbyte_cdk.sources.declarative.interpolation.filters import filters
-from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
-from airbyte_cdk.sources.declarative.interpolation.macros import macros
-from airbyte_cdk.sources.types import Config
 from jinja2 import meta
 from jinja2.environment import Template
 from jinja2.exceptions import UndefinedError
 from jinja2.sandbox import SandboxedEnvironment
 
+from airbyte_cdk.sources.declarative.interpolation.filters import filters
+from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
+from airbyte_cdk.sources.declarative.interpolation.macros import macros
+from airbyte_cdk.sources.types import Config
+
 
 class StreamPartitionAccessEnvironment(SandboxedEnvironment):
     """

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -8,9 +8,12 @@ import pkgutil
 from copy import deepcopy
 from importlib import metadata
 from typing import Any, Dict, Iterator, List, Mapping, Optional
-from packaging.version import Version, InvalidVersion
 
 import yaml
+from jsonschema.exceptions import ValidationError
+from jsonschema.validators import validate
+from packaging.version import InvalidVersion, Version
+
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
     AirbyteMessage,
@@ -44,8 +47,6 @@ from airbyte_cdk.sources.utils.slice_logger import (
     DebugSliceLogger,
     SliceLogger,
 )
-from jsonschema.exceptions import ValidationError
-from jsonschema.validators import validate
 
 
 class ManifestDeclarativeSource(DeclarativeSource):

airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py

@@ -5,9 +5,9 @@ from typing import Any, Mapping
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
 from airbyte_cdk.sources.declarative.models import (
+    CustomIncrementalSync,
     DatetimeBasedCursor,
     SubstreamPartitionRouter,
-    CustomIncrementalSync,
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig