airbyte-cdk 6.7.0rc3__py3-none-any.whl → 6.7.1.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -748,6 +748,123 @@ class NoPagination(BaseModel):
     type: Literal["NoPagination"]
 
 
+class State(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    min: int
+    max: int
+
+
+class OauthConnectorInputSpecification(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    consent_url: str = Field(
+        ...,
+        description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
+        examples=[
+            {
+                "consent_url": "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}"
+            },
+            {
+                "consent_url": "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}"
+            },
+        ],
+        title="DeclarativeOAuth Consent URL",
+    )
+    scope: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
+        examples=[{"scope": "user:read user:read_orders workspaces:read"}],
+        title="(Optional) DeclarativeOAuth Scope",
+    )
+    access_token_url: str = Field(
+        ...,
+        description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
+        examples=[
+            {
+                "access_token_url": "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
+            }
+        ],
+        title="DeclarativeOAuth Access Token URL",
+    )
+    access_token_headers: Optional[Dict[str, Any]] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
+        examples=[
+            {
+                "access_token_headers": {
+                    "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
+                }
+            }
+        ],
+        title="(Optional) DeclarativeOAuth Access Token Headers",
+    )
+    access_token_params: Optional[Dict[str, Any]] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
+        examples=[
+            {
+                "access_token_params": {
+                    "{auth_code_key}": "{{auth_code_key}}",
+                    "{client_id_key}": "{{client_id_key}}",
+                    "{client_secret_key}": "{{client_secret_key}}",
+                }
+            }
+        ],
+        title="(Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)",
+    )
+    extract_output: List[str] = Field(
+        ...,
+        description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.                ",
+        examples=[{"extract_output": ["access_token", "refresh_token", "other_field"]}],
+        title="DeclarativeOAuth Extract Output",
+    )
+    state: Optional[State] = Field(
+        None,
+        description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.                ",
+        examples=[{"state": {"min": 7, "max": 128}}],
+        title="(Optional) DeclarativeOAuth Configurable State Query Param",
+    )
+    client_id_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
+        examples=[{"client_id_key": "my_custom_client_id_key_name"}],
+        title="(Optional) DeclarativeOAuth Client ID Key Override",
+    )
+    client_secret_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
+        examples=[{"client_secret_key": "my_custom_client_secret_key_name"}],
+        title="(Optional) DeclarativeOAuth Client Secret Key Override",
+    )
+    scope_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
+        examples=[{"scope_key": "my_custom_scope_key_key_name"}],
+        title="(Optional) DeclarativeOAuth Scope Key Override",
+    )
+    state_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.                ",
+        examples=[{"state_key": "my_custom_state_key_key_name"}],
+        title="(Optional) DeclarativeOAuth State Key Override",
+    )
+    auth_code_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.                ",
+        examples=[{"auth_code_key": "my_custom_auth_code_key_name"}],
+        title="(Optional) DeclarativeOAuth Auth Code Key Override",
+    )
+    redirect_uri_key: Optional[str] = Field(
+        None,
+        description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
+        examples=[{"redirect_uri_key": "my_custom_redirect_uri_key_name"}],
+        title="(Optional) DeclarativeOAuth Redirect URI Key Override",
+    )
+
+
 class OAuthConfigSpecification(BaseModel):
     class Config:
         extra = Extra.allow
@@ -766,6 +883,11 @@ class OAuthConfigSpecification(BaseModel):
         ],
         title="OAuth user input",
     )
+    oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
+        None,
+        description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n  + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n  + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n  + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n  + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n  + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n  - The TikTok Marketing DeclarativeOAuth spec:\n  {\n    "oauth_connector_input_specification": {\n      "type": "object",\n      "additionalProperties": false,\n      "properties": {\n          "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n          "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n          "access_token_params": {\n              "{auth_code_key}": "{{auth_code_key}}",\n              "{client_id_key}": "{{client_id_key}}",\n              "{client_secret_key}": "{{client_secret_key}}"\n          },\n          "access_token_headers": {\n              "Content-Type": "application/json",\n              "Accept": "application/json"\n          },\n          "extract_output": ["data.access_token"],\n          "client_id_key": "app_id",\n          "client_secret_key": "secret",\n          "auth_code_key": "auth_code"\n      }\n    }\n  }',
+        title="DeclarativeOAuth Connector Specification",
+    )
     complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
         None,
         description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n    complete_oauth_output_specification={\n      refresh_token: {\n        type: string,\n        path_in_connector_config: ['credentials', 'refresh_token']\n      }\n    }",

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -17,6 +17,7 @@ from typing import (
     Mapping,
     MutableMapping,
     Optional,
+    Tuple,
     Type,
     Union,
     get_args,
@@ -24,6 +25,9 @@ from typing import (
     get_type_hints,
 )
 
+from isodate import parse_duration
+from pydantic.v1 import BaseModel
+
 from airbyte_cdk.models import FailureType, Level
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
@@ -366,8 +370,6 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_sta
 from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
 from airbyte_cdk.sources.types import Config
 from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
-from isodate import parse_duration
-from pydantic.v1 import BaseModel
 
 ComponentDefinition = Mapping[str, Any]
 
@@ -382,6 +384,7 @@ class ModelToComponentFactory:
         emit_connector_builder_messages: bool = False,
         disable_retries: bool = False,
         disable_cache: bool = False,
+        disable_resumable_full_refresh: bool = False,
         message_repository: Optional[MessageRepository] = None,
     ):
         self._init_mappings()
@@ -390,6 +393,7 @@ class ModelToComponentFactory:
         self._emit_connector_builder_messages = emit_connector_builder_messages
         self._disable_retries = disable_retries
         self._disable_cache = disable_cache
+        self._disable_resumable_full_refresh = disable_resumable_full_refresh
         self._message_repository = message_repository or InMemoryMessageRepository(  # type: ignore
             self._evaluate_log_level(emit_connector_builder_messages)
         )
@@ -752,7 +756,7 @@ class ModelToComponentFactory:
         config: Config,
         stream_state: MutableMapping[str, Any],
         **kwargs: Any,
-    ) -> ConcurrentCursor:
+    ) -> Tuple[ConcurrentCursor, DateTimeStreamStateConverter]:
         component_type = component_definition.get("type")
         if component_definition.get("type") != model_type.__name__:
             raise ValueError(
@@ -883,20 +887,23 @@ class ModelToComponentFactory:
             if evaluated_step:
                 step_length = parse_duration(evaluated_step)
 
-        return ConcurrentCursor(
-            stream_name=stream_name,
-            stream_namespace=stream_namespace,
-            stream_state=stream_state,
-            message_repository=self._message_repository,  # type: ignore  # message_repository is always instantiated with a value by factory
-            connector_state_manager=state_manager,
-            connector_state_converter=connector_state_converter,
-            cursor_field=cursor_field,
-            slice_boundary_fields=slice_boundary_fields,
-            start=start_date,  # type: ignore  # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
-            end_provider=end_date_provider,  # type: ignore  # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
-            lookback_window=lookback_window,
-            slice_range=step_length,
-            cursor_granularity=cursor_granularity,
+        return (
+            ConcurrentCursor(
+                stream_name=stream_name,
+                stream_namespace=stream_namespace,
+                stream_state=stream_state,
+                message_repository=self._message_repository,  # type: ignore  # message_repository is always instantiated with a value by factory
+                connector_state_manager=state_manager,
+                connector_state_converter=connector_state_converter,
+                cursor_field=cursor_field,
+                slice_boundary_fields=slice_boundary_fields,
+                start=start_date,  # type: ignore  # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
+                end_provider=end_date_provider,  # type: ignore  # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
+                lookback_window=lookback_window,
+                slice_range=step_length,
+                cursor_granularity=cursor_granularity,
+            ),
+            connector_state_converter,
         )
 
     @staticmethod
@@ -1330,6 +1337,8 @@ class ModelToComponentFactory:
                 if model.incremental_sync
                 else None
             )
+        elif self._disable_resumable_full_refresh:
+            return stream_slicer
         elif stream_slicer:
             # For the Full-Refresh sub-streams, we use the nested `ChildPartitionResumableFullRefreshCursor`
             return PerPartitionCursor(
@@ -1776,6 +1785,7 @@ class ModelToComponentFactory:
         self,
         model: RecordSelectorModel,
         config: Config,
+        name: str,
         *,
         transformations: List[RecordTransformation],
         decoder: Optional[Decoder] = None,
@@ -1806,6 +1816,7 @@ class ModelToComponentFactory:
 
         return RecordSelector(
             extractor=extractor,
+            name=name,
             config=config,
             record_filter=record_filter,
             transformations=transformations,
@@ -1876,6 +1887,7 @@ class ModelToComponentFactory:
         )
         record_selector = self._create_component_from_model(
             model=model.record_selector,
+            name=name,
             config=config,
             decoder=decoder,
             transformations=transformations,
@@ -2030,6 +2042,7 @@ class ModelToComponentFactory:
             requester=download_requester,
             record_selector=RecordSelector(
                 extractor=ResponseToFileExtractor(),
+                name=name,
                 record_filter=None,
                 transformations=[],
                 schema_normalization=TypeTransformer(TransformConfig.NoTransform),

airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py

@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
 from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
 
 import dpath
+
 from airbyte_cdk.models import AirbyteMessage
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
 from airbyte_cdk.sources.types import Config

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
 from airbyte_cdk.sources.types import Config

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py

@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (

airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py

@@ -9,6 +9,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
     get_numeric_value_from_header,

airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, List, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
 from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
     ErrorResolution,

airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
 from typing import Any, List, Mapping, MutableMapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import (
     DefaultHttpResponseFilter,
 )

airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py

@@ -5,6 +5,7 @@
 from typing import Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import (
     HttpResponseFilter,
 )

airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Set, Union
 
 import requests
+
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean

airbyte_cdk/sources/declarative/requesters/http_job_repository.py

@@ -6,6 +6,8 @@ from datetime import timedelta
 from typing import Any, Dict, Iterable, Mapping, Optional
 
 import requests
+from requests import Response
+
 from airbyte_cdk import AirbyteMessage
 from airbyte_cdk.logger import lazy_log
 from airbyte_cdk.models import FailureType, Type
@@ -23,7 +25,6 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
 from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
 from airbyte_cdk.sources.types import Record, StreamSlice
 from airbyte_cdk.utils import AirbyteTracedException
-from requests import Response
 
 LOGGER = logging.getLogger("airbyte")
 

airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -9,6 +9,7 @@ from typing import Any, Callable, Mapping, MutableMapping, Optional, Union
 from urllib.parse import urljoin
 
 import requests
+
 from airbyte_cdk.sources.declarative.auth.declarative_authenticator import (
     DeclarativeAuthenticator,
     NoAuth,

airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
 from typing import Any, Mapping, MutableMapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.decoders import (
     Decoder,
     JsonDecoder,

airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, MutableMapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
 from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
 

airbyte_cdk/sources/declarative/requesters/paginators/paginator.py

@@ -7,6 +7,7 @@ from dataclasses import dataclass
 from typing import Any, Mapping, Optional
 
 import requests
+
 from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
     RequestOptionsProvider,
 )

airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
 from typing import Any, Dict, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.decoders import (
     Decoder,
     JsonDecoder,

airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.decoders import (
     Decoder,
     JsonDecoder,

airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
     PaginationStrategy,

airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py

@@ -7,6 +7,7 @@ from dataclasses import dataclass
 from typing import Any, Optional
 
 import requests
+
 from airbyte_cdk.sources.types import Record
 
 

airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py

@@ -6,10 +6,12 @@ from abc import ABC, abstractmethod
 from typing import Any, Optional
 
 import requests
+
 from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
 from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
     PaginationStrategy,
 )
+from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor
 from airbyte_cdk.sources.types import Record
 
 
@@ -25,7 +27,11 @@ class PaginationStopCondition(ABC):
 
 
 class CursorStopCondition(PaginationStopCondition):
-    def __init__(self, cursor: DeclarativeCursor):
+    def __init__(
+        self,
+        cursor: DeclarativeCursor
+        | ConcurrentCursor,  # migrate to use both old and concurrent versions
+    ):
         self._cursor = cursor
 
     def is_met(self, record: Record) -> bool:
@@ -46,8 +52,8 @@ class StopConditionPaginationStrategyDecorator(PaginationStrategy):
             return None
         return self._delegate.next_page_token(response, last_page_size, last_record)
 
-    def reset(self) -> None:
-        self._delegate.reset()
+    def reset(self, reset_value: Optional[Any] = None) -> None:
+        self._delegate.reset(reset_value)
 
     def get_page_size(self) -> Optional[int]:
         return self._delegate.get_page_size()

airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py

@@ -5,6 +5,8 @@
 from dataclasses import InitVar, dataclass, field
 from typing import Any, Mapping, MutableMapping, Optional, Union
 
+from deprecated import deprecated
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping import NestedMapping
 from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_nested_request_input_provider import (
     InterpolatedNestedRequestInputProvider,
@@ -17,7 +19,6 @@ from airbyte_cdk.sources.declarative.requesters.request_options.request_options_
 )
 from airbyte_cdk.sources.source import ExperimentalClassWarning
 from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
-from deprecated import deprecated
 
 RequestInput = Union[str, Mapping[str, str]]
 ValidRequestTypes = (str, list)

airbyte_cdk/sources/declarative/requesters/requester.py

@@ -7,6 +7,7 @@ from enum import Enum
 from typing import Any, Callable, Mapping, MutableMapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
 from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
     RequestOptionsProvider,

airbyte_cdk/sources/declarative/retrievers/async_retriever.py

@@ -4,6 +4,8 @@
 from dataclasses import InitVar, dataclass, field
 from typing import Any, Callable, Iterable, Mapping, Optional
 
+from deprecated.classic import deprecated
+
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.async_job.job_orchestrator import (
     AsyncJobOrchestrator,
@@ -17,7 +19,6 @@ from airbyte_cdk.sources.source import ExperimentalClassWarning
 from airbyte_cdk.sources.streams.core import StreamData
 from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from deprecated.classic import deprecated
 
 
 @deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)

airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -20,6 +20,7 @@ from typing import (
 )
 
 import requests
+
 from airbyte_cdk.models import AirbyteMessage
 from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
 from airbyte_cdk.sources.declarative.incremental import ResumableFullRefreshCursor
@@ -467,8 +468,9 @@ class SimpleRetriever(Retriever):
         else:
             return None
 
-    @staticmethod
-    def _extract_record(stream_data: StreamData, stream_slice: StreamSlice) -> Optional[Record]:
+    def _extract_record(
+        self, stream_data: StreamData, stream_slice: StreamSlice
+    ) -> Optional[Record]:
         """
         As we allow the output of _read_pages to be StreamData, it can be multiple things. Therefore, we need to filter out and normalize
         to data to streamline the rest of the process.
@@ -477,9 +479,15 @@ class SimpleRetriever(Retriever):
             # Record is not part of `StreamData` but is the most common implementation of `Mapping[str, Any]` which is part of `StreamData`
             return stream_data
         elif isinstance(stream_data, (dict, Mapping)):
-            return Record(dict(stream_data), stream_slice)
+            return Record(
+                data=dict(stream_data), associated_slice=stream_slice, stream_name=self.name
+            )
         elif isinstance(stream_data, AirbyteMessage) and stream_data.record:
-            return Record(stream_data.record.data, stream_slice)
+            return Record(
+                data=stream_data.record.data,  # type:ignore # AirbyteMessage always has record.data
+                associated_slice=stream_slice,
+                stream_name=self.name,
+            )
         return None
 
     # stream_slices is defined with arguments on http stream and fixing this has a long tail of dependencies. Will be resolved by the decoupling of http stream and simple retriever

airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py

@@ -1,14 +1,13 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
-from typing import Iterable, Optional, Mapping, Any, Callable
+from typing import Any, Callable, Iterable, Mapping, Optional
 
 from airbyte_cdk.sources.declarative.retrievers import Retriever
 from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
 from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
-from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
 from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import StreamSlicer
-from airbyte_cdk.sources.types import StreamSlice
+from airbyte_cdk.sources.types import Record, StreamSlice
 from airbyte_cdk.utils.slice_hasher import SliceHasher
 
 
@@ -59,7 +58,11 @@ class DeclarativePartition(Partition):
     def read(self) -> Iterable[Record]:
         for stream_data in self._retriever.read_records(self._json_schema, self._stream_slice):
             if isinstance(stream_data, Mapping):
-                yield Record(stream_data, self)
+                yield Record(
+                    data=stream_data,
+                    stream_name=self.stream_name(),
+                    associated_slice=self._stream_slice,
+                )
             else:
                 self._message_repository.emit_message(stream_data)
 

airbyte_cdk/sources/declarative/transformations/add_fields.py

@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
 from typing import Any, Dict, List, Mapping, Optional, Type, Union
 
 import dpath
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.types import Config, FieldPointer, StreamSlice, StreamState

airbyte_cdk/sources/declarative/transformations/remove_fields.py

@@ -7,6 +7,7 @@ from typing import Any, Dict, List, Mapping, Optional
 
 import dpath
 import dpath.exceptions
+
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.types import Config, FieldPointer, StreamSlice, StreamState

airbyte_cdk/sources/declarative/yaml_declarative_source.py

@@ -6,6 +6,7 @@ import pkgutil
 from typing import Any, List, Mapping, Optional
 
 import yaml
+
 from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog
 from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
     ConcurrentDeclarativeSource,

airbyte_cdk/sources/embedded/tools.py

@@ -5,6 +5,7 @@
 from typing import Any, Callable, Dict, Iterable, Optional
 
 import dpath
+
 from airbyte_cdk.models import AirbyteStream
 
 

airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py

@@ -7,10 +7,11 @@ from abc import abstractmethod
 from typing import Any, Dict, List, Literal, Optional, Union
 
 import dpath
+from pydantic.v1 import AnyUrl, BaseModel, Field
+
 from airbyte_cdk import OneOfOptionConfig
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.utils import schema_helpers
-from pydantic.v1 import AnyUrl, BaseModel, Field
 
 
 class DeliverRecords(BaseModel):