airbyte-cdk 6.7.0rc2__py3-none-any.whl → 6.7.1__py3-none-any.whl

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -3,7 +3,7 @@
 #
 
 import logging
-from typing import Any, Generic, Iterator, List, Mapping, Optional, Tuple, Union, Callable
+from typing import Any, Callable, Generic, Iterator, List, Mapping, Optional, Tuple, Union
 
 from airbyte_cdk.models import (
     AirbyteCatalog,
@@ -27,14 +27,16 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 )
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     DatetimeBasedCursor as DatetimeBasedCursorModel,
+)
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     DeclarativeStream as DeclarativeStreamModel,
 )
 from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
-    ModelToComponentFactory,
     ComponentDefinition,
+    ModelToComponentFactory,
 )
 from airbyte_cdk.sources.declarative.requesters import HttpRequester
-from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever, Retriever
+from airbyte_cdk.sources.declarative.retrievers import Retriever, SimpleRetriever
 from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator import (
     DeclarativePartitionFactory,
     StreamSlicerPartitionGenerator,
@@ -42,15 +44,14 @@ from airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_genera
 from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields
 from airbyte_cdk.sources.declarative.types import ConnectionDefinition
 from airbyte_cdk.sources.source import TState
-from airbyte_cdk.sources.types import Config, StreamState
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
 from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
     AlwaysAvailableAvailabilityStrategy,
 )
-from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, FinalStateCursor
 from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream
+from airbyte_cdk.sources.types import Config, StreamState
 
 
 class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
@@ -194,44 +195,31 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                     declarative_stream.name
                 ].get("incremental_sync")
 
-                is_without_partition_router_nor_cursor = not bool(
+                if (
                     datetime_based_cursor_component_definition
-                ) and not (
-                    name_to_stream_mapping[declarative_stream.name]
-                    .get("retriever", {})
-                    .get("partition_router")
-                )
-                is_datetime_incremental_without_partition_routing = (
-                    self._is_datetime_incremental_without_partition_routing(
-                        datetime_based_cursor_component_definition, declarative_stream
+                    and datetime_based_cursor_component_definition.get("type", "")
+                    == DatetimeBasedCursorModel.__name__
+                    and self._stream_supports_concurrent_partition_processing(
+                        declarative_stream=declarative_stream
                     )
-                )
-                if (
-                    is_without_partition_router_nor_cursor
-                    or is_datetime_incremental_without_partition_routing
+                    and hasattr(declarative_stream.retriever, "stream_slicer")
+                    and isinstance(declarative_stream.retriever.stream_slicer, DatetimeBasedCursor)
                 ):
                     stream_state = state_manager.get_stream_state(
                         stream_name=declarative_stream.name, namespace=declarative_stream.namespace
                     )
 
-                    if is_datetime_incremental_without_partition_routing:
-                        cursor: Cursor = (
-                            self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
-                                state_manager=state_manager,
-                                model_type=DatetimeBasedCursorModel,
-                                component_definition=datetime_based_cursor_component_definition,
-                                stream_name=declarative_stream.name,
-                                stream_namespace=declarative_stream.namespace,
-                                config=config or {},
-                                stream_state=stream_state,
-                            )
-                        )
-                    else:
-                        cursor = FinalStateCursor(
-                            declarative_stream.name,
-                            declarative_stream.namespace,
-                            self.message_repository,
+                    cursor, connector_state_converter = (
+                        self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
+                            state_manager=state_manager,
+                            model_type=DatetimeBasedCursorModel,
+                            component_definition=datetime_based_cursor_component_definition,
+                            stream_name=declarative_stream.name,
+                            stream_namespace=declarative_stream.namespace,
+                            config=config or {},
+                            stream_state=stream_state,
                         )
+                    )
 
                     partition_generator = StreamSlicerPartitionGenerator(
                         DeclarativePartitionFactory(
@@ -254,9 +242,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             json_schema=declarative_stream.get_json_schema(),
                             availability_strategy=AlwaysAvailableAvailabilityStrategy(),
                             primary_key=get_primary_key_from_stream(declarative_stream.primary_key),
-                            cursor_field=cursor.cursor_field.cursor_field_key
-                            if hasattr(cursor, "cursor_field")
-                            else None,
+                            cursor_field=cursor.cursor_field.cursor_field_key,
                             logger=self.logger,
                             cursor=cursor,
                         )
@@ -268,22 +254,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
 
         return concurrent_streams, synchronous_streams
 
-    def _is_datetime_incremental_without_partition_routing(
-        self,
-        datetime_based_cursor_component_definition: Mapping[str, Any],
-        declarative_stream: DeclarativeStream,
-    ) -> bool:
-        return (
-            bool(datetime_based_cursor_component_definition)
-            and datetime_based_cursor_component_definition.get("type", "")
-            == DatetimeBasedCursorModel.__name__
-            and self._stream_supports_concurrent_partition_processing(
-                declarative_stream=declarative_stream
-            )
-            and hasattr(declarative_stream.retriever, "stream_slicer")
-            and isinstance(declarative_stream.retriever.stream_slicer, DatetimeBasedCursor)
-        )
-
     def _stream_supports_concurrent_partition_processing(
         self, declarative_stream: DeclarativeStream
     ) -> bool:

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1972,6 +1972,173 @@ definitions:
           - app_id:
               type: string
               path_in_connector_config: ["info", "app_id"]
+      oauth_connector_input_specification:
+        title: DeclarativeOAuth Connector Specification
+        description: |-
+          The DeclarativeOAuth specific blob.
+          Pertains to the fields defined by the connector relating to the OAuth flow.
+
+          Interpolation capabilities:
+          - The variables placeholders are declared as `{my_var}`.
+          - The nested resolution variables like `{{my_nested_var}}` is allowed as well.
+
+          - The allowed interpolation context is:
+            + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}
+            + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}
+            + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}
+            + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}
+            + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}
+
+          Examples:
+            - The TikTok Marketing DeclarativeOAuth spec:
+            {
+              "oauth_connector_input_specification": {
+                "type": "object",
+                "additionalProperties": false,
+                "properties": {
+                    "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
+                    "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",
+                    "access_token_params": {
+                        "{auth_code_key}": "{{auth_code_key}}",
+                        "{client_id_key}": "{{client_id_key}}",
+                        "{client_secret_key}": "{{client_secret_key}}"
+                    },
+                    "access_token_headers": {
+                        "Content-Type": "application/json",
+                        "Accept": "application/json"
+                    },
+                    "extract_output": ["data.access_token"],
+                    "client_id_key": "app_id",
+                    "client_secret_key": "secret",
+                    "auth_code_key": "auth_code"
+                }
+              }
+            }
+        type: object
+        additionalProperties: true
+        required:
+          - consent_url
+          - access_token_url
+          - extract_output
+        properties:
+          consent_url:
+            title: DeclarativeOAuth Consent URL
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific string URL string template to initiate the authentication.
+              The placeholders are replaced during the processing to provide neccessary values.
+            examples:
+              - consent_url: https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}
+              - consent_url: https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}
+          scope:
+            title: (Optional) DeclarativeOAuth Scope
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.
+            examples:
+              - scope: user:read user:read_orders workspaces:read
+          access_token_url:
+            title: DeclarativeOAuth Access Token URL
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.
+              The placeholders are replaced during the processing to provide neccessary values.
+            examples:
+              - access_token_url: https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}} 
+          access_token_headers:
+            title: (Optional) DeclarativeOAuth Access Token Headers
+            type: object
+            additionalProperties: true
+            description: |-
+              The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
+            examples:
+              - access_token_headers: {
+                  "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
+                }
+          access_token_params:
+            title: (Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)
+            type: object
+            additionalProperties: true
+            description: |-
+              The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
+              When this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.
+            examples:
+              - access_token_params: {
+                  "{auth_code_key}": "{{auth_code_key}}",
+                  "{client_id_key}": "{{client_id_key}}",
+                  "{client_secret_key}": "{{client_secret_key}}"
+                }
+          extract_output:
+            title: DeclarativeOAuth Extract Output
+            type: array
+            items:
+              type: string
+            description: |-
+              The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.                
+            examples:
+              - extract_output: ["access_token", "refresh_token", "other_field"]
+          state:
+            title: (Optional) DeclarativeOAuth Configurable State Query Param
+            type: object
+            additionalProperties: true
+            required:
+              - min
+              - max
+            description: |-
+              The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,
+              including length and complexity.                
+            properties:
+              min:
+                type: integer
+              max:
+                type: integer
+            examples:
+              - state: {
+                  "min": 7,
+                  "max": 128,
+                }
+          client_id_key:
+            title: (Optional) DeclarativeOAuth Client ID Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.
+            examples:
+              - client_id_key: "my_custom_client_id_key_name"
+          client_secret_key:
+            title: (Optional) DeclarativeOAuth Client Secret Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.
+            examples:
+              - client_secret_key: "my_custom_client_secret_key_name"
+          scope_key:
+            title: (Optional) DeclarativeOAuth Scope Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.
+            examples:
+              - scope_key: "my_custom_scope_key_key_name"
+          state_key:
+            title: (Optional) DeclarativeOAuth State Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.                
+            examples:
+              - state_key: "my_custom_state_key_key_name"
+          auth_code_key:
+            title: (Optional) DeclarativeOAuth Auth Code Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.                
+            examples:
+              - auth_code_key: "my_custom_auth_code_key_name"
+          redirect_uri_key:
+            title: (Optional) DeclarativeOAuth Redirect URI Key Override
+            type: string
+            description: |-
+              The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.
+            examples:
+              - redirect_uri_key: "my_custom_redirect_uri_key_name"
       complete_oauth_output_specification:
         title: "OAuth output specification"
         description: |-
@@ -2790,21 +2957,21 @@ interpolation:
         - created_at: "2020-01-01 00:00:00.000+00:00"
         - updated_at: "2020-01-02 00:00:00.000+00:00"
   macros:
-    - title: Now (UTC)
+    - title: now_utc
       description: Returns the current date and time in the UTC timezone.
       arguments: {}
       return_type: Datetime
       examples:
         - "'{{ now_utc() }}' -> '2021-09-01 00:00:00+00:00'"
         - "'{{ now_utc().strftime('%Y-%m-%d') }}' -> '2021-09-01'"
-    - title: Today (UTC)
+    - title: today_utc
       description: Returns the current date in UTC timezone. The output is a date object.
       arguments: {}
       return_type: Date
       examples:
         - "'{{ today_utc() }}' -> '2021-09-01'"
         - "'{{ today_utc().strftime('%Y/%m/%d')}}' -> '2021/09/01'"
-    - title: Timestamp
+    - title: timestamp
       description: Converts a number or a string representing a datetime (formatted as ISO8601) to a timestamp. If the input is a number, it is converted to an int. If no timezone is specified, the string is interpreted as UTC.
       arguments:
         datetime: A string formatted as ISO8601 or an integer representing a unix timestamp
@@ -2815,7 +2982,7 @@ interpolation:
         - "'{{ timestamp('2022-02-28T00:00:00Z') }}' -> 1646006400"
         - "'{{ timestamp('2022-02-28 00:00:00Z') }}' -> 1646006400"
         - "'{{ timestamp('2022-02-28T00:00:00-08:00') }}' -> 1646035200"
-    - title: Max
+    - title: max
       description: Returns the largest object of a iterable, or or two or more arguments.
       arguments:
         args: iterable or a sequence of two or more arguments
@@ -2823,7 +2990,7 @@ interpolation:
       examples:
         - "'{{ max(2, 3) }}' -> 3"
         - "'{{ max([2, 3]) }}' -> 3"
-    - title: Day Delta
+    - title: day_delta
       description: Returns the datetime of now() + num_days.
       arguments:
         num_days: The number of days to add to now
@@ -2833,8 +3000,8 @@ interpolation:
         - "'{{ day_delta(1) }}' -> '2021-09-02T00:00:00.000000+0000'"
         - "'{{ day_delta(-1) }}' -> '2021-08-31:00:00.000000+0000'"
         - "'{{ day_delta(25, format='%Y-%m-%d') }}' -> '2021-09-02'"
-    - title: Duration
-      description: Converts an ISO8601 duratioin to datetime.timedelta.
+    - title: duration
+      description: Converts an ISO8601 duration to datetime timedelta.
       arguments:
         duration_string: "A string representing an ISO8601 duration. See https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm for more details."
       return_type: datetime.timedelta
@@ -2842,7 +3009,7 @@ interpolation:
         - "'{{ duration('P1D') }}' -> '1 day, 0:00:00'"
         - "'{{ duration('P6DT23H') }}' -> '6 days, 23:00:00'"
         - "'{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}' -> '2021-08-31T00:00:00Z'"
-    - title: Format Datetime
+    - title: format_datetime
       description: Converts a datetime or a datetime-string to the specified format.
       arguments:
         datetime: The datetime object or a string to convert. If datetime is a string, it must be formatted as ISO8601.
@@ -2854,7 +3021,7 @@ interpolation:
         - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ') }}"
         - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ', '%a, %d %b %Y %H:%M:%S %z') }}"
   filters:
-    - title: Hash
+    - title: hash
       description: Convert the specified value to a hashed string.
       arguments:
         hash_type: Valid hash type for converts ('md5' as default value).
@@ -2864,26 +3031,26 @@ interpolation:
         - "{{ 'Test client_secret' | hash() }} -> '3032d57a12f76b61a820e47b9a5a0cbb'"
         - "{{ 'Test client_secret' | hash('md5') }} -> '3032d57a12f76b61a820e47b9a5a0cbb'"
         - "{{ 'Test client_secret' | hash('md5', salt='salt') }} -> '5011a0168579c2d94cbbe1c6ad14327c'"
-    - title: Base64 encoder
+    - title: base64encode
       description: Convert the specified value to a string in the base64 format.
       arguments: {}
       return_type: str
       examples:
         - "{{ 'Test client_secret' | base64encode }} -> 'VGVzdCBjbGllbnRfc2VjcmV0'"
-    - title: Base64 decoder
+    - title: base64decode
       description: Decodes the specified base64 format value into a common string.
       arguments: {}
       return_type: str
       examples:
         - "{{ 'ZmFrZSByZWZyZXNoX3Rva2VuIHZhbHVl' | base64decode }} -> 'fake refresh_token value'"
-    - title: String
+    - title: string
       description: Converts the specified value to a string.
       arguments: {}
       return_type: str
       examples:
         - '{{ 1 | string }} -> "1"'
         - '{{ ["hello", "world" | string }} -> "["hello", "world"]"'
-    - title: Regex Search
+    - title: regex_search
       description: Match the input string against a regular expression and return the first match.
       arguments:
         regex: The regular expression to search for. It must include a capture group.

airbyte_cdk/sources/declarative/decoders/json_decoder.py

@@ -5,11 +5,12 @@ import codecs
 import logging
 from dataclasses import InitVar, dataclass
 from gzip import decompress
-from typing import Any, Generator, Mapping, MutableMapping, List, Optional
+from typing import Any, Generator, List, Mapping, MutableMapping, Optional
 
+import orjson
 import requests
+
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
-import orjson
 
 logger = logging.getLogger("airbyte")
 

airbyte_cdk/sources/declarative/decoders/noop_decoder.py

@@ -4,6 +4,7 @@ import logging
 from typing import Any, Generator, Mapping
 
 import requests
+
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
 
 logger = logging.getLogger("airbyte")

airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py

@@ -7,6 +7,7 @@ from dataclasses import dataclass
 from typing import Any, Generator, MutableMapping
 
 import requests
+
 from airbyte_cdk.sources.declarative.decoders import Decoder
 
 logger = logging.getLogger("airbyte")

airbyte_cdk/sources/declarative/decoders/xml_decoder.py

@@ -9,6 +9,7 @@ from xml.parsers.expat import ExpatError
 
 import requests
 import xmltodict
+
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
 
 logger = logging.getLogger("airbyte")

airbyte_cdk/sources/declarative/extractors/dpath_extractor.py

@@ -7,6 +7,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Union
 
 import dpath
 import requests
+
 from airbyte_cdk.sources.declarative.decoders import Decoder, JsonDecoder
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString

airbyte_cdk/sources/declarative/extractors/http_selector.py

@@ -6,6 +6,7 @@ from abc import abstractmethod
 from typing import Any, Iterable, Mapping, Optional
 
 import requests
+
 from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
 
 

airbyte_cdk/sources/declarative/extractors/record_filter.py

@@ -1,7 +1,6 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-import datetime
 from dataclasses import InitVar, dataclass
 from typing import Any, Iterable, Mapping, Optional, Union
 
@@ -11,7 +10,7 @@ from airbyte_cdk.sources.declarative.incremental import (
     PerPartitionWithGlobalCursor,
 )
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
-from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
 
 
 @dataclass
@@ -68,20 +67,6 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
         self._date_time_based_cursor = date_time_based_cursor
         self._substream_cursor = substream_cursor
 
-    @property
-    def _cursor_field(self) -> str:
-        return self._date_time_based_cursor.cursor_field.eval(self._date_time_based_cursor.config)  # type: ignore # eval returns a string in this context
-
-    @property
-    def _start_date_from_config(self) -> datetime.datetime:
-        return self._date_time_based_cursor._start_datetime.get_datetime(
-            self._date_time_based_cursor.config
-        )
-
-    @property
-    def _end_datetime(self) -> datetime.datetime:
-        return self._date_time_based_cursor.select_best_end_datetime()
-
     def filter_records(
         self,
         records: Iterable[Mapping[str, Any]],
@@ -89,16 +74,14 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
     ) -> Iterable[Mapping[str, Any]]:
-        state_value = self._get_state_value(
-            stream_state, stream_slice or StreamSlice(partition={}, cursor_slice={})
-        )
-        filter_date: datetime.datetime = self._get_filter_date(state_value)
         records = (
             record
             for record in records
-            if self._end_datetime
-            >= self._date_time_based_cursor.parse_date(record[self._cursor_field])
-            >= filter_date
+            if (self._substream_cursor or self._date_time_based_cursor).should_be_synced(
+                # Record is created on the fly to align with cursors interface; stream name is ignored as we don't need it here
+                # Record stream name is empty cause it is not used durig the filtering
+                Record(data=record, associated_slice=stream_slice, stream_name="")
+            )
         )
         if self.condition:
             records = super().filter_records(
@@ -108,28 +91,3 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
                 next_page_token=next_page_token,
             )
         yield from records
-
-    def _get_state_value(
-        self, stream_state: StreamState, stream_slice: StreamSlice
-    ) -> Optional[str]:
-        """
-        Return cursor_value or None in case it was not found.
-        Cursor_value may be empty if:
-            1. It is an initial sync => no stream_state exist at all.
-            2. In Parent-child stream, and we already make initial sync, so stream_state is present.
-               During the second read, we receive one extra record from parent and therefore no stream_state for this record will be found.
-
-        :param StreamState stream_state: State
-        :param StreamSlice stream_slice: Current Stream slice
-        :return Optional[str]: cursor_value in case it was found, otherwise None.
-        """
-        state = (self._substream_cursor or self._date_time_based_cursor).select_state(stream_slice)
-
-        return state.get(self._cursor_field) if state else None
-
-    def _get_filter_date(self, state_value: Optional[str]) -> datetime.datetime:
-        start_date_parsed = self._start_date_from_config
-        if state_value:
-            return max(start_date_parsed, self._date_time_based_cursor.parse_date(state_value))
-        else:
-            return start_date_parsed

airbyte_cdk/sources/declarative/extractors/record_selector.py

@@ -3,12 +3,14 @@
 #
 
 from dataclasses import InitVar, dataclass, field
-from typing import Any, Iterable, List, Mapping, Optional
+from typing import Any, Iterable, List, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.models import SchemaNormalization
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
@@ -37,11 +39,34 @@ class RecordSelector(HttpSelector):
     config: Config
     parameters: InitVar[Mapping[str, Any]]
     schema_normalization: TypeTransformer
+    name: str
+    _name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
     record_filter: Optional[RecordFilter] = None
     transformations: List[RecordTransformation] = field(default_factory=lambda: [])
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters
+        self._name = (
+            InterpolatedString(self._name, parameters=parameters)
+            if isinstance(self._name, str)
+            else self._name
+        )
+
+    @property  # type: ignore
+    def name(self) -> str:
+        """
+        :return: Stream name
+        """
+        return (
+            str(self._name.eval(self.config))
+            if isinstance(self._name, InterpolatedString)
+            else self._name
+        )
+
+    @name.setter
+    def name(self, value: str) -> None:
+        if not isinstance(value, property):
+            self._name = value
 
     def select_records(
         self,
@@ -85,7 +110,7 @@ class RecordSelector(HttpSelector):
         transformed_data = self._transform(filtered_data, stream_state, stream_slice)
         normalized_data = self._normalize_by_schema(transformed_data, schema=records_schema)
         for data in normalized_data:
-            yield Record(data, stream_slice)
+            yield Record(data=data, stream_name=self.name, associated_slice=stream_slice)
 
     def _normalize_by_schema(
         self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]
@@ -125,6 +150,9 @@ class RecordSelector(HttpSelector):
         for record in records:
             for transformation in self.transformations:
                 transformation.transform(
-                    record, config=self.config, stream_state=stream_state, stream_slice=stream_slice
-                )  # type: ignore  # record has type Mapping[str, Any], but Dict[str, Any] expected
+                    record,  # type: ignore  # record has type Mapping[str, Any], but Dict[str, Any] expected
+                    config=self.config,
+                    stream_state=stream_state,
+                    stream_slice=stream_slice,
+                )
             yield record

airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py

@@ -10,9 +10,10 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
 
 import pandas as pd
 import requests
-from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from numpy import nan
 
+from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
+
 EMPTY_STR: str = ""
 DEFAULT_ENCODING: str = "utf-8"
 DOWNLOAD_CHUNK_SIZE: int = 1024 * 10