airbyte-cdk 6.31.1__py3-none-any.whl → 6.31.2.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of airbyte-cdk might be problematic. Click here for more details.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +3 -9
- airbyte_cdk/connector_builder/connector_builder_handler.py +2 -3
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +4 -4
- airbyte_cdk/sources/declarative/auth/jwt.py +11 -17
- airbyte_cdk/sources/declarative/auth/oauth.py +23 -89
- airbyte_cdk/sources/declarative/auth/token.py +3 -8
- airbyte_cdk/sources/declarative/auth/token_provider.py +5 -4
- airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +9 -19
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +43 -134
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +16 -55
- airbyte_cdk/sources/declarative/declarative_stream.py +1 -3
- airbyte_cdk/sources/declarative/extractors/record_filter.py +5 -3
- airbyte_cdk/sources/declarative/incremental/__init__.py +0 -6
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +7 -6
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +0 -3
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +3 -35
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +7 -15
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +15 -45
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +64 -343
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +5 -5
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +4 -2
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +15 -55
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +0 -22
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +4 -4
- airbyte_cdk/sources/declarative/requesters/http_requester.py +5 -1
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +6 -5
- airbyte_cdk/sources/declarative/requesters/request_option.py +83 -4
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +7 -6
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +12 -6
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +5 -2
- airbyte_cdk/sources/declarative/schema/__init__.py +0 -2
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +5 -44
- airbyte_cdk/sources/http_logger.py +1 -1
- airbyte_cdk/sources/streams/concurrent/cursor.py +57 -51
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +13 -22
- airbyte_cdk/sources/streams/core.py +6 -6
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +62 -231
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +88 -171
- airbyte_cdk/sources/types.py +2 -4
- airbyte_cdk/sources/utils/transform.py +2 -23
- airbyte_cdk/test/utils/manifest_only_fixtures.py +2 -1
- airbyte_cdk/utils/mapping_helpers.py +86 -27
- airbyte_cdk/utils/slice_hasher.py +1 -8
- {airbyte_cdk-6.31.1.dist-info → airbyte_cdk-6.31.2.dev0.dist-info}/METADATA +6 -6
- {airbyte_cdk-6.31.1.dist-info → airbyte_cdk-6.31.2.dev0.dist-info}/RECORD +48 -54
- {airbyte_cdk-6.31.1.dist-info → airbyte_cdk-6.31.2.dev0.dist-info}/WHEEL +1 -1
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +0 -400
- airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +0 -143
- airbyte_cdk/sources/streams/concurrent/clamping.py +0 -99
- airbyte_cdk/sources/streams/concurrent/cursor_types.py +0 -32
- airbyte_cdk/utils/datetime_helpers.py +0 -499
- airbyte_cdk-6.31.1.dist-info/LICENSE_SHORT +0 -1
- {airbyte_cdk-6.31.1.dist-info → airbyte_cdk-6.31.2.dev0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.31.1.dist-info → airbyte_cdk-6.31.2.dev0.dist-info}/entry_points.txt +0 -0
| @@ -4,9 +4,9 @@ from dataclasses import InitVar, dataclass, field | |
| 4 4 | 
             
            from typing import Any, Callable, Iterable, Mapping, Optional
         | 
| 5 5 |  | 
| 6 6 | 
             
            from airbyte_cdk.models import FailureType
         | 
| 7 | 
            -
            from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
         | 
| 8 7 | 
             
            from airbyte_cdk.sources.declarative.async_job.job_orchestrator import (
         | 
| 9 8 | 
             
                AsyncJobOrchestrator,
         | 
| 9 | 
            +
                AsyncPartition,
         | 
| 10 10 | 
             
            )
         | 
| 11 11 | 
             
            from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import (
         | 
| 12 12 | 
             
                SinglePartitionRouter,
         | 
| @@ -42,12 +42,12 @@ class AsyncJobPartitionRouter(StreamSlicer): | |
| 42 42 |  | 
| 43 43 | 
             
                    for completed_partition in self._job_orchestrator.create_and_get_completed_partitions():
         | 
| 44 44 | 
             
                        yield StreamSlice(
         | 
| 45 | 
            -
                            partition=dict(completed_partition.stream_slice.partition) | 
| 45 | 
            +
                            partition=dict(completed_partition.stream_slice.partition)
         | 
| 46 | 
            +
                            | {"partition": completed_partition},
         | 
| 46 47 | 
             
                            cursor_slice=completed_partition.stream_slice.cursor_slice,
         | 
| 47 | 
            -
                            extra_fields={"jobs": list(completed_partition.jobs)},
         | 
| 48 48 | 
             
                        )
         | 
| 49 49 |  | 
| 50 | 
            -
                def fetch_records(self,  | 
| 50 | 
            +
                def fetch_records(self, partition: AsyncPartition) -> Iterable[Mapping[str, Any]]:
         | 
| 51 51 | 
             
                    """
         | 
| 52 52 | 
             
                    This method of fetching records extends beyond what a PartitionRouter/StreamSlicer should
         | 
| 53 53 | 
             
                    be responsible for. However, this was added in because the JobOrchestrator is required to
         | 
| @@ -62,4 +62,4 @@ class AsyncJobPartitionRouter(StreamSlicer): | |
| 62 62 | 
             
                            failure_type=FailureType.system_error,
         | 
| 63 63 | 
             
                        )
         | 
| 64 64 |  | 
| 65 | 
            -
                    return self._job_orchestrator.fetch_records( | 
| 65 | 
            +
                    return self._job_orchestrator.fetch_records(partition=partition)
         | 
| @@ -3,7 +3,7 @@ | |
| 3 3 | 
             
            #
         | 
| 4 4 |  | 
| 5 5 | 
             
            from dataclasses import InitVar, dataclass
         | 
| 6 | 
            -
            from typing import Any, Iterable, List, Mapping, Optional, Union
         | 
| 6 | 
            +
            from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
         | 
| 7 7 |  | 
| 8 8 | 
             
            from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
         | 
| 9 9 | 
             
            from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
         | 
| @@ -100,7 +100,9 @@ class ListPartitionRouter(PartitionRouter): | |
| 100 100 | 
             
                    ):
         | 
| 101 101 | 
             
                        slice_value = stream_slice.get(self._cursor_field.eval(self.config))
         | 
| 102 102 | 
             
                        if slice_value:
         | 
| 103 | 
            -
                             | 
| 103 | 
            +
                            options: MutableMapping[str, Any] = {}
         | 
| 104 | 
            +
                            self.request_option.inject_into_request(options, slice_value, self.config)
         | 
| 105 | 
            +
                            return options
         | 
| 104 106 | 
             
                        else:
         | 
| 105 107 | 
             
                            return {}
         | 
| 106 108 | 
             
                    else:
         | 
| @@ -4,7 +4,7 @@ | |
| 4 4 | 
             
            import copy
         | 
| 5 5 | 
             
            import logging
         | 
| 6 6 | 
             
            from dataclasses import InitVar, dataclass
         | 
| 7 | 
            -
            from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
         | 
| 7 | 
            +
            from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
         | 
| 8 8 |  | 
| 9 9 | 
             
            import dpath
         | 
| 10 10 |  | 
| @@ -118,7 +118,7 @@ class SubstreamPartitionRouter(PartitionRouter): | |
| 118 118 | 
             
                def _get_request_option(
         | 
| 119 119 | 
             
                    self, option_type: RequestOptionType, stream_slice: Optional[StreamSlice]
         | 
| 120 120 | 
             
                ) -> Mapping[str, Any]:
         | 
| 121 | 
            -
                    params = {}
         | 
| 121 | 
            +
                    params: MutableMapping[str, Any] = {}
         | 
| 122 122 | 
             
                    if stream_slice:
         | 
| 123 123 | 
             
                        for parent_config in self.parent_stream_configs:
         | 
| 124 124 | 
             
                            if (
         | 
| @@ -128,13 +128,7 @@ class SubstreamPartitionRouter(PartitionRouter): | |
| 128 128 | 
             
                                key = parent_config.partition_field.eval(self.config)  # type: ignore # partition_field is always casted to an interpolated string
         | 
| 129 129 | 
             
                                value = stream_slice.get(key)
         | 
| 130 130 | 
             
                                if value:
         | 
| 131 | 
            -
                                    params. | 
| 132 | 
            -
                                        {
         | 
| 133 | 
            -
                                            parent_config.request_option.field_name.eval(  # type: ignore [union-attr]
         | 
| 134 | 
            -
                                                config=self.config
         | 
| 135 | 
            -
                                            ): value
         | 
| 136 | 
            -
                                        }
         | 
| 137 | 
            -
                                    )
         | 
| 131 | 
            +
                                    parent_config.request_option.inject_into_request(params, value, self.config)
         | 
| 138 132 | 
             
                    return params
         | 
| 139 133 |  | 
| 140 134 | 
             
                def stream_slices(self) -> Iterable[StreamSlice]:
         | 
| @@ -295,58 +289,24 @@ class SubstreamPartitionRouter(PartitionRouter): | |
| 295 289 | 
             
                        return
         | 
| 296 290 |  | 
| 297 291 | 
             
                    if not parent_state and incremental_dependency:
         | 
| 298 | 
            -
                        #  | 
| 299 | 
            -
                         | 
| 292 | 
            +
                        # Attempt to retrieve child state
         | 
| 293 | 
            +
                        substream_state = list(stream_state.values())
         | 
| 294 | 
            +
                        substream_state = substream_state[0] if substream_state else {}  # type: ignore [assignment]  # Incorrect type for assignment
         | 
| 295 | 
            +
                        parent_state = {}
         | 
| 296 | 
            +
             | 
| 297 | 
            +
                        # Copy child state to parent streams with incremental dependencies
         | 
| 298 | 
            +
                        if substream_state:
         | 
| 299 | 
            +
                            for parent_config in self.parent_stream_configs:
         | 
| 300 | 
            +
                                if parent_config.incremental_dependency:
         | 
| 301 | 
            +
                                    parent_state[parent_config.stream.name] = {
         | 
| 302 | 
            +
                                        parent_config.stream.cursor_field: substream_state
         | 
| 303 | 
            +
                                    }
         | 
| 300 304 |  | 
| 301 305 | 
             
                    # Set state for each parent stream with an incremental dependency
         | 
| 302 306 | 
             
                    for parent_config in self.parent_stream_configs:
         | 
| 303 307 | 
             
                        if parent_config.incremental_dependency:
         | 
| 304 308 | 
             
                            parent_config.stream.state = parent_state.get(parent_config.stream.name, {})
         | 
| 305 309 |  | 
| 306 | 
            -
                def _migrate_child_state_to_parent_state(self, stream_state: StreamState) -> StreamState:
         | 
| 307 | 
            -
                    """
         | 
| 308 | 
            -
                    Migrate the child stream state to the parent stream's state format.
         | 
| 309 | 
            -
             | 
| 310 | 
            -
                    This method converts the global or child state into a format compatible with parent
         | 
| 311 | 
            -
                    streams. The migration occurs only for parent streams with incremental dependencies.
         | 
| 312 | 
            -
                    The method filters out per-partition states and retains only the global state in the
         | 
| 313 | 
            -
                    format `{cursor_field: cursor_value}`.
         | 
| 314 | 
            -
             | 
| 315 | 
            -
                    Args:
         | 
| 316 | 
            -
                        stream_state (StreamState): The state to migrate. Expected formats include:
         | 
| 317 | 
            -
                            - {"updated_at": "2023-05-27T00:00:00Z"}
         | 
| 318 | 
            -
                            - {"states": [...] } (ignored during migration)
         | 
| 319 | 
            -
             | 
| 320 | 
            -
                    Returns:
         | 
| 321 | 
            -
                        StreamState: A migrated state for parent streams in the format:
         | 
| 322 | 
            -
                            {
         | 
| 323 | 
            -
                                "parent_stream_name": {"parent_stream_cursor": "2023-05-27T00:00:00Z"}
         | 
| 324 | 
            -
                            }
         | 
| 325 | 
            -
             | 
| 326 | 
            -
                    Example:
         | 
| 327 | 
            -
                        Input: {"updated_at": "2023-05-27T00:00:00Z"}
         | 
| 328 | 
            -
                        Output: {
         | 
| 329 | 
            -
                            "parent_stream_name": {"parent_stream_cursor": "2023-05-27T00:00:00Z"}
         | 
| 330 | 
            -
                        }
         | 
| 331 | 
            -
                    """
         | 
| 332 | 
            -
                    substream_state_values = list(stream_state.values())
         | 
| 333 | 
            -
                    substream_state = substream_state_values[0] if substream_state_values else {}
         | 
| 334 | 
            -
             | 
| 335 | 
            -
                    # Ignore per-partition states or invalid formats
         | 
| 336 | 
            -
                    if isinstance(substream_state, (list, dict)) or len(substream_state_values) != 1:
         | 
| 337 | 
            -
                        return {}
         | 
| 338 | 
            -
             | 
| 339 | 
            -
                    # Copy child state to parent streams with incremental dependencies
         | 
| 340 | 
            -
                    parent_state = {}
         | 
| 341 | 
            -
                    if substream_state:
         | 
| 342 | 
            -
                        for parent_config in self.parent_stream_configs:
         | 
| 343 | 
            -
                            if parent_config.incremental_dependency:
         | 
| 344 | 
            -
                                parent_state[parent_config.stream.name] = {
         | 
| 345 | 
            -
                                    parent_config.stream.cursor_field: substream_state
         | 
| 346 | 
            -
                                }
         | 
| 347 | 
            -
             | 
| 348 | 
            -
                    return parent_state
         | 
| 349 | 
            -
             | 
| 350 310 | 
             
                def get_stream_state(self) -> Optional[Mapping[str, StreamState]]:
         | 
| 351 311 | 
             
                    """
         | 
| 352 312 | 
             
                    Get the state of the parent streams.
         | 
| @@ -8,7 +8,6 @@ from typing import Any, List, Mapping, Optional, Union | |
| 8 8 | 
             
            import requests
         | 
| 9 9 |  | 
| 10 10 | 
             
            from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
         | 
| 11 | 
            -
            from airbyte_cdk.sources.streams.http.error_handlers.backoff_strategy import BackoffStrategy
         | 
| 12 11 | 
             
            from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
         | 
| 13 12 | 
             
                ErrorResolution,
         | 
| 14 13 | 
             
                ResponseAction,
         | 
| @@ -78,24 +77,3 @@ class CompositeErrorHandler(ErrorHandler): | |
| 78 77 | 
             
                        return matched_error_resolution
         | 
| 79 78 |  | 
| 80 79 | 
             
                    return create_fallback_error_resolution(response_or_exception)
         | 
| 81 | 
            -
             | 
| 82 | 
            -
                @property
         | 
| 83 | 
            -
                def backoff_strategies(self) -> Optional[List[BackoffStrategy]]:
         | 
| 84 | 
            -
                    """
         | 
| 85 | 
            -
                    Combines backoff strategies from all child error handlers into a single flattened list.
         | 
| 86 | 
            -
             | 
| 87 | 
            -
                    When used with HttpRequester, note the following behavior:
         | 
| 88 | 
            -
                    - In HttpRequester.__post_init__, the entire list of backoff strategies is assigned to the error handler
         | 
| 89 | 
            -
                    - However, the error handler's backoff_time() method only ever uses the first non-None strategy in the list
         | 
| 90 | 
            -
                    - This means that if any backoff strategies are present, the first non-None strategy becomes the default
         | 
| 91 | 
            -
                    - This applies to both user-defined response filters and errors from DEFAULT_ERROR_MAPPING
         | 
| 92 | 
            -
                    - The list structure is not used to map different strategies to different error conditions
         | 
| 93 | 
            -
                    - Therefore, subsequent strategies in the list will not be used
         | 
| 94 | 
            -
             | 
| 95 | 
            -
                    Returns None if no handlers have strategies defined, which will result in HttpRequester using its default backoff strategy.
         | 
| 96 | 
            -
                    """
         | 
| 97 | 
            -
                    all_strategies = []
         | 
| 98 | 
            -
                    for handler in self.error_handlers:
         | 
| 99 | 
            -
                        if hasattr(handler, "backoff_strategies") and handler.backoff_strategies:
         | 
| 100 | 
            -
                            all_strategies.extend(handler.backoff_strategies)
         | 
| 101 | 
            -
                    return all_strategies if all_strategies else None
         | 
| @@ -151,16 +151,16 @@ class HttpResponseFilter: | |
| 151 151 | 
             
                    :param response: The HTTP response which can be used during interpolation
         | 
| 152 152 | 
             
                    :return: The evaluated error message string to be emitted
         | 
| 153 153 | 
             
                    """
         | 
| 154 | 
            -
                    return self.error_message.eval(  # type: ignore[no-any-return, union-attr]
         | 
| 154 | 
            +
                    return self.error_message.eval(  # type: ignore [no-any-return, union-attr]
         | 
| 155 155 | 
             
                        self.config, response=self._safe_response_json(response), headers=response.headers
         | 
| 156 156 | 
             
                    )
         | 
| 157 157 |  | 
| 158 158 | 
             
                def _response_matches_predicate(self, response: requests.Response) -> bool:
         | 
| 159 159 | 
             
                    return (
         | 
| 160 160 | 
             
                        bool(
         | 
| 161 | 
            -
                            self.predicate.condition  # type:ignore[union-attr]
         | 
| 162 | 
            -
                            and self.predicate.eval(  # type:ignore[union-attr]
         | 
| 163 | 
            -
                                None,  # type: ignore[arg-type]
         | 
| 161 | 
            +
                            self.predicate.condition  # type: ignore [union-attr]
         | 
| 162 | 
            +
                            and self.predicate.eval(  # type: ignore [union-attr]
         | 
| 163 | 
            +
                                None,  # type: ignore [arg-type]
         | 
| 164 164 | 
             
                                response=self._safe_response_json(response),
         | 
| 165 165 | 
             
                                headers=response.headers,
         | 
| 166 166 | 
             
                            )
         | 
| @@ -199,6 +199,9 @@ class HttpRequester(Requester): | |
| 199 199 | 
             
                    Raise a ValueError if there's a key collision
         | 
| 200 200 | 
             
                    Returned merged mapping otherwise
         | 
| 201 201 | 
             
                    """
         | 
| 202 | 
            +
             | 
| 203 | 
            +
                    is_body_json = requester_method.__name__ == "get_request_body_json"
         | 
| 204 | 
            +
             | 
| 202 205 | 
             
                    return combine_mappings(
         | 
| 203 206 | 
             
                        [
         | 
| 204 207 | 
             
                            requester_method(
         | 
| @@ -208,7 +211,8 @@ class HttpRequester(Requester): | |
| 208 211 | 
             
                            ),
         | 
| 209 212 | 
             
                            auth_options_method(),
         | 
| 210 213 | 
             
                            extra_options,
         | 
| 211 | 
            -
                        ]
         | 
| 214 | 
            +
                        ],
         | 
| 215 | 
            +
                        allow_same_value_merge=is_body_json,
         | 
| 212 216 | 
             
                    )
         | 
| 213 217 |  | 
| 214 218 | 
             
                def _request_headers(
         | 
| @@ -187,7 +187,7 @@ class DefaultPaginator(Paginator): | |
| 187 187 | 
             
                def _get_request_options(
         | 
| 188 188 | 
             
                    self, option_type: RequestOptionType, next_page_token: Optional[Mapping[str, Any]]
         | 
| 189 189 | 
             
                ) -> MutableMapping[str, Any]:
         | 
| 190 | 
            -
                    options = {}
         | 
| 190 | 
            +
                    options: MutableMapping[str, Any] = {}
         | 
| 191 191 |  | 
| 192 192 | 
             
                    token = next_page_token.get("next_page_token") if next_page_token else None
         | 
| 193 193 | 
             
                    if (
         | 
| @@ -196,15 +196,16 @@ class DefaultPaginator(Paginator): | |
| 196 196 | 
             
                        and isinstance(self.page_token_option, RequestOption)
         | 
| 197 197 | 
             
                        and self.page_token_option.inject_into == option_type
         | 
| 198 198 | 
             
                    ):
         | 
| 199 | 
            -
                         | 
| 199 | 
            +
                        self.page_token_option.inject_into_request(options, token, self.config)
         | 
| 200 | 
            +
             | 
| 200 201 | 
             
                    if (
         | 
| 201 202 | 
             
                        self.page_size_option
         | 
| 202 203 | 
             
                        and self.pagination_strategy.get_page_size()
         | 
| 203 204 | 
             
                        and self.page_size_option.inject_into == option_type
         | 
| 204 205 | 
             
                    ):
         | 
| 205 | 
            -
                         | 
| 206 | 
            -
             | 
| 207 | 
            -
             | 
| 206 | 
            +
                        page_size = self.pagination_strategy.get_page_size()
         | 
| 207 | 
            +
                        self.page_size_option.inject_into_request(options, page_size, self.config)
         | 
| 208 | 
            +
             | 
| 208 209 | 
             
                    return options
         | 
| 209 210 |  | 
| 210 211 |  | 
| @@ -4,9 +4,10 @@ | |
| 4 4 |  | 
| 5 5 | 
             
            from dataclasses import InitVar, dataclass
         | 
| 6 6 | 
             
            from enum import Enum
         | 
| 7 | 
            -
            from typing import Any, Mapping, Union
         | 
| 7 | 
            +
            from typing import Any, List, Literal, Mapping, MutableMapping, Optional, Union
         | 
| 8 8 |  | 
| 9 9 | 
             
            from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
         | 
| 10 | 
            +
            from airbyte_cdk.sources.types import Config
         | 
| 10 11 |  | 
| 11 12 |  | 
| 12 13 | 
             
            class RequestOptionType(Enum):
         | 
| @@ -26,13 +27,91 @@ class RequestOption: | |
| 26 27 | 
             
                Describes an option to set on a request
         | 
| 27 28 |  | 
| 28 29 | 
             
                Attributes:
         | 
| 29 | 
            -
                    field_name (str): Describes the name of the parameter to inject
         | 
| 30 | 
            +
                    field_name (str): Describes the name of the parameter to inject. Mutually exclusive with field_path.
         | 
| 31 | 
            +
                    field_path (list(str)): Describes the path to a nested field as a list of field names.
         | 
| 32 | 
            +
                      Only valid for body_json injection type, and mutually exclusive with field_name.
         | 
| 30 33 | 
             
                    inject_into (RequestOptionType): Describes where in the HTTP request to inject the parameter
         | 
| 31 34 | 
             
                """
         | 
| 32 35 |  | 
| 33 | 
            -
                field_name: Union[InterpolatedString, str]
         | 
| 34 36 | 
             
                inject_into: RequestOptionType
         | 
| 35 37 | 
             
                parameters: InitVar[Mapping[str, Any]]
         | 
| 38 | 
            +
                field_name: Optional[Union[InterpolatedString, str]] = None
         | 
| 39 | 
            +
                field_path: Optional[List[Union[InterpolatedString, str]]] = None
         | 
| 36 40 |  | 
| 37 41 | 
             
                def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         | 
| 38 | 
            -
                     | 
| 42 | 
            +
                    # Validate inputs. We should expect either field_name or field_path, but not both
         | 
| 43 | 
            +
                    if self.field_name is None and self.field_path is None:
         | 
| 44 | 
            +
                        raise ValueError("RequestOption requires either a field_name or field_path")
         | 
| 45 | 
            +
             | 
| 46 | 
            +
                    if self.field_name is not None and self.field_path is not None:
         | 
| 47 | 
            +
                        raise ValueError(
         | 
| 48 | 
            +
                            "Only one of field_name or field_path can be provided to RequestOption"
         | 
| 49 | 
            +
                        )
         | 
| 50 | 
            +
             | 
| 51 | 
            +
                    # Nested field injection is only supported for body JSON injection
         | 
| 52 | 
            +
                    if self.field_path is not None and self.inject_into != RequestOptionType.body_json:
         | 
| 53 | 
            +
                        raise ValueError(
         | 
| 54 | 
            +
                            "Nested field injection is only supported for body JSON injection. Please use a top-level field_name for other injection types."
         | 
| 55 | 
            +
                        )
         | 
| 56 | 
            +
             | 
| 57 | 
            +
                    # Convert field_name and field_path into InterpolatedString objects if they are strings
         | 
| 58 | 
            +
                    if self.field_name is not None:
         | 
| 59 | 
            +
                        self.field_name = InterpolatedString.create(self.field_name, parameters=parameters)
         | 
| 60 | 
            +
                    elif self.field_path is not None:
         | 
| 61 | 
            +
                        self.field_path = [
         | 
| 62 | 
            +
                            InterpolatedString.create(segment, parameters=parameters)
         | 
| 63 | 
            +
                            for segment in self.field_path
         | 
| 64 | 
            +
                        ]
         | 
| 65 | 
            +
             | 
| 66 | 
            +
                @property
         | 
| 67 | 
            +
                def _is_field_path(self) -> bool:
         | 
| 68 | 
            +
                    """Returns whether this option is a field path (ie, a nested field)"""
         | 
| 69 | 
            +
                    return self.field_path is not None
         | 
| 70 | 
            +
             | 
| 71 | 
            +
                def inject_into_request(
         | 
| 72 | 
            +
                    self,
         | 
| 73 | 
            +
                    target: MutableMapping[str, Any],
         | 
| 74 | 
            +
                    value: Any,
         | 
| 75 | 
            +
                    config: Config,
         | 
| 76 | 
            +
                ) -> None:
         | 
| 77 | 
            +
                    """
         | 
| 78 | 
            +
                    Inject a request option value into a target request structure using either field_name or field_path.
         | 
| 79 | 
            +
                    For non-body-json injection, only top-level field names are supported.
         | 
| 80 | 
            +
                    For body-json injection, both field names and nested field paths are supported.
         | 
| 81 | 
            +
             | 
| 82 | 
            +
                    Args:
         | 
| 83 | 
            +
                        target: The request structure to inject the value into
         | 
| 84 | 
            +
                        value: The value to inject
         | 
| 85 | 
            +
                        config: The config object to use for interpolation
         | 
| 86 | 
            +
                    """
         | 
| 87 | 
            +
                    if self._is_field_path:
         | 
| 88 | 
            +
                        if self.inject_into != RequestOptionType.body_json:
         | 
| 89 | 
            +
                            raise ValueError(
         | 
| 90 | 
            +
                                "Nested field injection is only supported for body JSON injection. Please use a top-level field_name for other injection types."
         | 
| 91 | 
            +
                            )
         | 
| 92 | 
            +
             | 
| 93 | 
            +
                        assert self.field_path is not None  # for type checker
         | 
| 94 | 
            +
                        current = target
         | 
| 95 | 
            +
                        # Convert path segments into strings, evaluating any interpolated segments
         | 
| 96 | 
            +
                        # Example: ["data", "{{ config[user_type] }}", "id"] -> ["data", "admin", "id"]
         | 
| 97 | 
            +
                        *path_parts, final_key = [
         | 
| 98 | 
            +
                            str(
         | 
| 99 | 
            +
                                segment.eval(config=config)
         | 
| 100 | 
            +
                                if isinstance(segment, InterpolatedString)
         | 
| 101 | 
            +
                                else segment
         | 
| 102 | 
            +
                            )
         | 
| 103 | 
            +
                            for segment in self.field_path
         | 
| 104 | 
            +
                        ]
         | 
| 105 | 
            +
             | 
| 106 | 
            +
                        # Build a nested dictionary structure and set the final value at the deepest level
         | 
| 107 | 
            +
                        for part in path_parts:
         | 
| 108 | 
            +
                            current = current.setdefault(part, {})
         | 
| 109 | 
            +
                        current[final_key] = value
         | 
| 110 | 
            +
                    else:
         | 
| 111 | 
            +
                        # For non-nested fields, evaluate the field name if it's an interpolated string
         | 
| 112 | 
            +
                        key = (
         | 
| 113 | 
            +
                            self.field_name.eval(config=config)
         | 
| 114 | 
            +
                            if isinstance(self.field_name, InterpolatedString)
         | 
| 115 | 
            +
                            else self.field_name
         | 
| 116 | 
            +
                        )
         | 
| 117 | 
            +
                        target[str(key)] = value
         | 
| @@ -80,12 +80,13 @@ class DatetimeBasedRequestOptionsProvider(RequestOptionsProvider): | |
| 80 80 | 
             
                    options: MutableMapping[str, Any] = {}
         | 
| 81 81 | 
             
                    if not stream_slice:
         | 
| 82 82 | 
             
                        return options
         | 
| 83 | 
            +
             | 
| 83 84 | 
             
                    if self.start_time_option and self.start_time_option.inject_into == option_type:
         | 
| 84 | 
            -
                         | 
| 85 | 
            -
             | 
| 86 | 
            -
             | 
| 85 | 
            +
                        start_time_value = stream_slice.get(self._partition_field_start.eval(self.config))
         | 
| 86 | 
            +
                        self.start_time_option.inject_into_request(options, start_time_value, self.config)
         | 
| 87 | 
            +
             | 
| 87 88 | 
             
                    if self.end_time_option and self.end_time_option.inject_into == option_type:
         | 
| 88 | 
            -
                         | 
| 89 | 
            -
             | 
| 90 | 
            -
             | 
| 89 | 
            +
                        end_time_value = stream_slice.get(self._partition_field_end.eval(self.config))
         | 
| 90 | 
            +
                        self.end_time_option.inject_into_request(options, end_time_value, self.config)
         | 
| 91 | 
            +
             | 
| 91 92 | 
             
                    return options
         | 
| @@ -6,7 +6,7 @@ from typing import Any, Iterable, Mapping, Optional | |
| 6 6 |  | 
| 7 7 | 
             
            from typing_extensions import deprecated
         | 
| 8 8 |  | 
| 9 | 
            -
            from airbyte_cdk. | 
| 9 | 
            +
            from airbyte_cdk.models import FailureType
         | 
| 10 10 | 
             
            from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncPartition
         | 
| 11 11 | 
             
            from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
         | 
| 12 12 | 
             
            from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
         | 
| @@ -16,6 +16,7 @@ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever | |
| 16 16 | 
             
            from airbyte_cdk.sources.source import ExperimentalClassWarning
         | 
| 17 17 | 
             
            from airbyte_cdk.sources.streams.core import StreamData
         | 
| 18 18 | 
             
            from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
         | 
| 19 | 
            +
            from airbyte_cdk.utils.traced_exception import AirbyteTracedException
         | 
| 19 20 |  | 
| 20 21 |  | 
| 21 22 | 
             
            @deprecated(
         | 
| @@ -56,9 +57,9 @@ class AsyncRetriever(Retriever): | |
| 56 57 |  | 
| 57 58 | 
             
                    return self.state
         | 
| 58 59 |  | 
| 59 | 
            -
                def  | 
| 60 | 
            +
                def _validate_and_get_stream_slice_partition(
         | 
| 60 61 | 
             
                    self, stream_slice: Optional[StreamSlice] = None
         | 
| 61 | 
            -
                ) ->  | 
| 62 | 
            +
                ) -> AsyncPartition:
         | 
| 62 63 | 
             
                    """
         | 
| 63 64 | 
             
                    Validates the stream_slice argument and returns the partition from it.
         | 
| 64 65 |  | 
| @@ -72,7 +73,12 @@ class AsyncRetriever(Retriever): | |
| 72 73 | 
             
                        AirbyteTracedException: If the stream_slice is not an instance of StreamSlice or if the partition is not present in the stream_slice.
         | 
| 73 74 |  | 
| 74 75 | 
             
                    """
         | 
| 75 | 
            -
                     | 
| 76 | 
            +
                    if not isinstance(stream_slice, StreamSlice) or "partition" not in stream_slice.partition:
         | 
| 77 | 
            +
                        raise AirbyteTracedException(
         | 
| 78 | 
            +
                            message="Invalid arguments to AsyncJobRetriever.read_records: stream_slice is no optional. Please contact Airbyte Support",
         | 
| 79 | 
            +
                            failure_type=FailureType.system_error,
         | 
| 80 | 
            +
                        )
         | 
| 81 | 
            +
                    return stream_slice["partition"]  # type: ignore  # stream_slice["partition"] has been added as an AsyncPartition as part of stream_slices
         | 
| 76 82 |  | 
| 77 83 | 
             
                def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
         | 
| 78 84 | 
             
                    return self.stream_slicer.stream_slices()
         | 
| @@ -83,8 +89,8 @@ class AsyncRetriever(Retriever): | |
| 83 89 | 
             
                    stream_slice: Optional[StreamSlice] = None,
         | 
| 84 90 | 
             
                ) -> Iterable[StreamData]:
         | 
| 85 91 | 
             
                    stream_state: StreamState = self._get_stream_state()
         | 
| 86 | 
            -
                     | 
| 87 | 
            -
                    records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records( | 
| 92 | 
            +
                    partition: AsyncPartition = self._validate_and_get_stream_slice_partition(stream_slice)
         | 
| 93 | 
            +
                    records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(partition)
         | 
| 88 94 |  | 
| 89 95 | 
             
                    yield from self.record_selector.filter_and_transform(
         | 
| 90 96 | 
             
                        all_data=records,
         | 
| @@ -128,6 +128,9 @@ class SimpleRetriever(Retriever): | |
| 128 128 | 
             
                    Returned merged mapping otherwise
         | 
| 129 129 | 
             
                    """
         | 
| 130 130 | 
             
                    # FIXME we should eventually remove the usage of stream_state as part of the interpolation
         | 
| 131 | 
            +
             | 
| 132 | 
            +
                    is_body_json = paginator_method.__name__ == "get_request_body_json"
         | 
| 133 | 
            +
             | 
| 131 134 | 
             
                    mappings = [
         | 
| 132 135 | 
             
                        paginator_method(
         | 
| 133 136 | 
             
                            stream_state=stream_state,
         | 
| @@ -143,7 +146,7 @@ class SimpleRetriever(Retriever): | |
| 143 146 | 
             
                                next_page_token=next_page_token,
         | 
| 144 147 | 
             
                            )
         | 
| 145 148 | 
             
                        )
         | 
| 146 | 
            -
                    return combine_mappings(mappings)
         | 
| 149 | 
            +
                    return combine_mappings(mappings, allow_same_value_merge=is_body_json)
         | 
| 147 150 |  | 
| 148 151 | 
             
                def _request_headers(
         | 
| 149 152 | 
             
                    self,
         | 
| @@ -160,7 +163,7 @@ class SimpleRetriever(Retriever): | |
| 160 163 | 
             
                        stream_slice,
         | 
| 161 164 | 
             
                        next_page_token,
         | 
| 162 165 | 
             
                        self._paginator.get_request_headers,
         | 
| 163 | 
            -
                        self. | 
| 166 | 
            +
                        self.stream_slicer.get_request_headers,
         | 
| 164 167 | 
             
                    )
         | 
| 165 168 | 
             
                    if isinstance(headers, str):
         | 
| 166 169 | 
             
                        raise ValueError("Request headers cannot be a string")
         | 
| @@ -4,7 +4,6 @@ | |
| 4 4 |  | 
| 5 5 | 
             
            from airbyte_cdk.sources.declarative.schema.default_schema_loader import DefaultSchemaLoader
         | 
| 6 6 | 
             
            from airbyte_cdk.sources.declarative.schema.dynamic_schema_loader import (
         | 
| 7 | 
            -
                ComplexFieldType,
         | 
| 8 7 | 
             
                DynamicSchemaLoader,
         | 
| 9 8 | 
             
                SchemaTypeIdentifier,
         | 
| 10 9 | 
             
                TypesMap,
         | 
| @@ -19,7 +18,6 @@ __all__ = [ | |
| 19 18 | 
             
                "SchemaLoader",
         | 
| 20 19 | 
             
                "InlineSchemaLoader",
         | 
| 21 20 | 
             
                "DynamicSchemaLoader",
         | 
| 22 | 
            -
                "ComplexFieldType",
         | 
| 23 21 | 
             
                "TypesMap",
         | 
| 24 22 | 
             
                "SchemaTypeIdentifier",
         | 
| 25 23 | 
             
            ]
         | 
| @@ -18,7 +18,7 @@ from airbyte_cdk.sources.declarative.transformations import RecordTransformation | |
| 18 18 | 
             
            from airbyte_cdk.sources.source import ExperimentalClassWarning
         | 
| 19 19 | 
             
            from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
         | 
| 20 20 |  | 
| 21 | 
            -
            AIRBYTE_DATA_TYPES: Mapping[str,  | 
| 21 | 
            +
            AIRBYTE_DATA_TYPES: Mapping[str, Mapping[str, Any]] = {
         | 
| 22 22 | 
             
                "string": {"type": ["null", "string"]},
         | 
| 23 23 | 
             
                "boolean": {"type": ["null", "boolean"]},
         | 
| 24 24 | 
             
                "date": {"type": ["null", "string"], "format": "date"},
         | 
| @@ -45,25 +45,6 @@ AIRBYTE_DATA_TYPES: Mapping[str, MutableMapping[str, Any]] = { | |
| 45 45 | 
             
            }
         | 
| 46 46 |  | 
| 47 47 |  | 
| 48 | 
            -
            @deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
         | 
| 49 | 
            -
            @dataclass(frozen=True)
         | 
| 50 | 
            -
            class ComplexFieldType:
         | 
| 51 | 
            -
                """
         | 
| 52 | 
            -
                Identifies complex field type
         | 
| 53 | 
            -
                """
         | 
| 54 | 
            -
             | 
| 55 | 
            -
                field_type: str
         | 
| 56 | 
            -
                items: Optional[Union[str, "ComplexFieldType"]] = None
         | 
| 57 | 
            -
             | 
| 58 | 
            -
                def __post_init__(self) -> None:
         | 
| 59 | 
            -
                    """
         | 
| 60 | 
            -
                    Enforces that `items` is only used when `field_type` is a array
         | 
| 61 | 
            -
                    """
         | 
| 62 | 
            -
                    # `items_type` is valid only for array target types
         | 
| 63 | 
            -
                    if self.items and self.field_type != "array":
         | 
| 64 | 
            -
                        raise ValueError("'items' can only be used when 'field_type' is an array.")
         | 
| 65 | 
            -
             | 
| 66 | 
            -
             | 
| 67 48 | 
             
            @deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
         | 
| 68 49 | 
             
            @dataclass(frozen=True)
         | 
| 69 50 | 
             
            class TypesMap:
         | 
| @@ -71,7 +52,7 @@ class TypesMap: | |
| 71 52 | 
             
                Represents a mapping between a current type and its corresponding target type.
         | 
| 72 53 | 
             
                """
         | 
| 73 54 |  | 
| 74 | 
            -
                target_type: Union[List[str], str | 
| 55 | 
            +
                target_type: Union[List[str], str]
         | 
| 75 56 | 
             
                current_type: Union[List[str], str]
         | 
| 76 57 | 
             
                condition: Optional[str]
         | 
| 77 58 |  | 
| @@ -154,9 +135,8 @@ class DynamicSchemaLoader(SchemaLoader): | |
| 154 135 | 
             
                    transformed_properties = self._transform(properties, {})
         | 
| 155 136 |  | 
| 156 137 | 
             
                    return {
         | 
| 157 | 
            -
                        "$schema": " | 
| 138 | 
            +
                        "$schema": "http://json-schema.org/draft-07/schema#",
         | 
| 158 139 | 
             
                        "type": "object",
         | 
| 159 | 
            -
                        "additionalProperties": True,
         | 
| 160 140 | 
             
                        "properties": transformed_properties,
         | 
| 161 141 | 
             
                    }
         | 
| 162 142 |  | 
| @@ -208,37 +188,18 @@ class DynamicSchemaLoader(SchemaLoader): | |
| 208 188 | 
             
                        first_type = self._get_airbyte_type(mapped_field_type[0])
         | 
| 209 189 | 
             
                        second_type = self._get_airbyte_type(mapped_field_type[1])
         | 
| 210 190 | 
             
                        return {"oneOf": [first_type, second_type]}
         | 
| 211 | 
            -
             | 
| 212 191 | 
             
                    elif isinstance(mapped_field_type, str):
         | 
| 213 192 | 
             
                        return self._get_airbyte_type(mapped_field_type)
         | 
| 214 | 
            -
             | 
| 215 | 
            -
                    elif isinstance(mapped_field_type, ComplexFieldType):
         | 
| 216 | 
            -
                        return self._resolve_complex_type(mapped_field_type)
         | 
| 217 | 
            -
             | 
| 218 193 | 
             
                    else:
         | 
| 219 194 | 
             
                        raise ValueError(
         | 
| 220 195 | 
             
                            f"Invalid data type. Available string or two items list of string. Got {mapped_field_type}."
         | 
| 221 196 | 
             
                        )
         | 
| 222 197 |  | 
| 223 | 
            -
                def _resolve_complex_type(self, complex_type: ComplexFieldType) -> Mapping[str, Any]:
         | 
| 224 | 
            -
                    if not complex_type.items:
         | 
| 225 | 
            -
                        return self._get_airbyte_type(complex_type.field_type)
         | 
| 226 | 
            -
             | 
| 227 | 
            -
                    field_type = self._get_airbyte_type(complex_type.field_type)
         | 
| 228 | 
            -
             | 
| 229 | 
            -
                    field_type["items"] = (
         | 
| 230 | 
            -
                        self._get_airbyte_type(complex_type.items)
         | 
| 231 | 
            -
                        if isinstance(complex_type.items, str)
         | 
| 232 | 
            -
                        else self._resolve_complex_type(complex_type.items)
         | 
| 233 | 
            -
                    )
         | 
| 234 | 
            -
             | 
| 235 | 
            -
                    return field_type
         | 
| 236 | 
            -
             | 
| 237 198 | 
             
                def _replace_type_if_not_valid(
         | 
| 238 199 | 
             
                    self,
         | 
| 239 200 | 
             
                    field_type: Union[List[str], str],
         | 
| 240 201 | 
             
                    raw_schema: MutableMapping[str, Any],
         | 
| 241 | 
            -
                ) -> Union[List[str], str | 
| 202 | 
            +
                ) -> Union[List[str], str]:
         | 
| 242 203 | 
             
                    """
         | 
| 243 204 | 
             
                    Replaces a field type if it matches a type mapping in `types_map`.
         | 
| 244 205 | 
             
                    """
         | 
| @@ -255,7 +216,7 @@ class DynamicSchemaLoader(SchemaLoader): | |
| 255 216 | 
             
                    return field_type
         | 
| 256 217 |  | 
| 257 218 | 
             
                @staticmethod
         | 
| 258 | 
            -
                def _get_airbyte_type(field_type: str) ->  | 
| 219 | 
            +
                def _get_airbyte_type(field_type: str) -> Mapping[str, Any]:
         | 
| 259 220 | 
             
                    """
         | 
| 260 221 | 
             
                    Maps a field type to its corresponding Airbyte type definition.
         | 
| 261 222 | 
             
                    """
         | 
| @@ -45,7 +45,7 @@ def format_http_message( | |
| 45 45 | 
             
                    log_message["http"]["is_auxiliary"] = is_auxiliary  # type: ignore [index]
         | 
| 46 46 | 
             
                if stream_name:
         | 
| 47 47 | 
             
                    log_message["airbyte_cdk"] = {"stream": {"name": stream_name}}
         | 
| 48 | 
            -
                return log_message  # type: ignore[return-value]  # got "dict[str, object]", expected "dict[str, JsonType]"
         | 
| 48 | 
            +
                return log_message  # type: ignore [return-value]  # got "dict[str, object]", expected "dict[str, JsonType]"
         | 
| 49 49 |  | 
| 50 50 |  | 
| 51 51 | 
             
            def _normalize_body_string(body_str: Optional[Union[str, bytes]]) -> Optional[str]:
         |