airbyte-cdk 6.31.1.dev0__py3-none-any.whl → 6.31.2.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +3 -9
- airbyte_cdk/connector_builder/connector_builder_handler.py +2 -3
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +4 -4
- airbyte_cdk/sources/declarative/auth/jwt.py +11 -17
- airbyte_cdk/sources/declarative/auth/oauth.py +23 -89
- airbyte_cdk/sources/declarative/auth/token.py +3 -8
- airbyte_cdk/sources/declarative/auth/token_provider.py +5 -4
- airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +9 -19
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +43 -134
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +16 -55
- airbyte_cdk/sources/declarative/declarative_stream.py +1 -3
- airbyte_cdk/sources/declarative/extractors/record_filter.py +5 -3
- airbyte_cdk/sources/declarative/incremental/__init__.py +0 -6
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +7 -6
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +0 -3
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +3 -35
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +7 -15
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +15 -45
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +64 -343
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +5 -5
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +4 -2
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +15 -55
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +0 -22
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +4 -4
- airbyte_cdk/sources/declarative/requesters/http_requester.py +5 -1
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +6 -5
- airbyte_cdk/sources/declarative/requesters/request_option.py +83 -4
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +7 -6
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +12 -6
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +5 -2
- airbyte_cdk/sources/declarative/schema/__init__.py +0 -2
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +5 -44
- airbyte_cdk/sources/http_logger.py +1 -1
- airbyte_cdk/sources/streams/concurrent/cursor.py +57 -51
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +13 -22
- airbyte_cdk/sources/streams/core.py +6 -6
- airbyte_cdk/sources/streams/http/http.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +6 -17
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +31 -43
- airbyte_cdk/sources/types.py +2 -4
- airbyte_cdk/sources/utils/transform.py +2 -23
- airbyte_cdk/test/utils/manifest_only_fixtures.py +2 -1
- airbyte_cdk/utils/mapping_helpers.py +86 -27
- airbyte_cdk/utils/slice_hasher.py +1 -8
- {airbyte_cdk-6.31.1.dev0.dist-info → airbyte_cdk-6.31.2.dev0.dist-info}/METADATA +6 -6
- {airbyte_cdk-6.31.1.dev0.dist-info → airbyte_cdk-6.31.2.dev0.dist-info}/RECORD +49 -55
- {airbyte_cdk-6.31.1.dev0.dist-info → airbyte_cdk-6.31.2.dev0.dist-info}/WHEEL +1 -1
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +0 -400
- airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +0 -143
- airbyte_cdk/sources/streams/concurrent/clamping.py +0 -99
- airbyte_cdk/sources/streams/concurrent/cursor_types.py +0 -32
- airbyte_cdk/utils/datetime_helpers.py +0 -499
- airbyte_cdk-6.31.1.dev0.dist-info/LICENSE_SHORT +0 -1
- {airbyte_cdk-6.31.1.dev0.dist-info → airbyte_cdk-6.31.2.dev0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.31.1.dev0.dist-info → airbyte_cdk-6.31.2.dev0.dist-info}/entry_points.txt +0 -0
@@ -4,9 +4,9 @@ from dataclasses import InitVar, dataclass, field
|
|
4
4
|
from typing import Any, Callable, Iterable, Mapping, Optional
|
5
5
|
|
6
6
|
from airbyte_cdk.models import FailureType
|
7
|
-
from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
|
8
7
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import (
|
9
8
|
AsyncJobOrchestrator,
|
9
|
+
AsyncPartition,
|
10
10
|
)
|
11
11
|
from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import (
|
12
12
|
SinglePartitionRouter,
|
@@ -42,12 +42,12 @@ class AsyncJobPartitionRouter(StreamSlicer):
|
|
42
42
|
|
43
43
|
for completed_partition in self._job_orchestrator.create_and_get_completed_partitions():
|
44
44
|
yield StreamSlice(
|
45
|
-
partition=dict(completed_partition.stream_slice.partition)
|
45
|
+
partition=dict(completed_partition.stream_slice.partition)
|
46
|
+
| {"partition": completed_partition},
|
46
47
|
cursor_slice=completed_partition.stream_slice.cursor_slice,
|
47
|
-
extra_fields={"jobs": list(completed_partition.jobs)},
|
48
48
|
)
|
49
49
|
|
50
|
-
def fetch_records(self,
|
50
|
+
def fetch_records(self, partition: AsyncPartition) -> Iterable[Mapping[str, Any]]:
|
51
51
|
"""
|
52
52
|
This method of fetching records extends beyond what a PartitionRouter/StreamSlicer should
|
53
53
|
be responsible for. However, this was added in because the JobOrchestrator is required to
|
@@ -62,4 +62,4 @@ class AsyncJobPartitionRouter(StreamSlicer):
|
|
62
62
|
failure_type=FailureType.system_error,
|
63
63
|
)
|
64
64
|
|
65
|
-
return self._job_orchestrator.fetch_records(
|
65
|
+
return self._job_orchestrator.fetch_records(partition=partition)
|
@@ -3,7 +3,7 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
from dataclasses import InitVar, dataclass
|
6
|
-
from typing import Any, Iterable, List, Mapping, Optional, Union
|
6
|
+
from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
8
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
9
9
|
from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
|
@@ -100,7 +100,9 @@ class ListPartitionRouter(PartitionRouter):
|
|
100
100
|
):
|
101
101
|
slice_value = stream_slice.get(self._cursor_field.eval(self.config))
|
102
102
|
if slice_value:
|
103
|
-
|
103
|
+
options: MutableMapping[str, Any] = {}
|
104
|
+
self.request_option.inject_into_request(options, slice_value, self.config)
|
105
|
+
return options
|
104
106
|
else:
|
105
107
|
return {}
|
106
108
|
else:
|
@@ -4,7 +4,7 @@
|
|
4
4
|
import copy
|
5
5
|
import logging
|
6
6
|
from dataclasses import InitVar, dataclass
|
7
|
-
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
|
7
|
+
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
|
8
8
|
|
9
9
|
import dpath
|
10
10
|
|
@@ -118,7 +118,7 @@ class SubstreamPartitionRouter(PartitionRouter):
|
|
118
118
|
def _get_request_option(
|
119
119
|
self, option_type: RequestOptionType, stream_slice: Optional[StreamSlice]
|
120
120
|
) -> Mapping[str, Any]:
|
121
|
-
params = {}
|
121
|
+
params: MutableMapping[str, Any] = {}
|
122
122
|
if stream_slice:
|
123
123
|
for parent_config in self.parent_stream_configs:
|
124
124
|
if (
|
@@ -128,13 +128,7 @@ class SubstreamPartitionRouter(PartitionRouter):
|
|
128
128
|
key = parent_config.partition_field.eval(self.config) # type: ignore # partition_field is always casted to an interpolated string
|
129
129
|
value = stream_slice.get(key)
|
130
130
|
if value:
|
131
|
-
params.
|
132
|
-
{
|
133
|
-
parent_config.request_option.field_name.eval( # type: ignore [union-attr]
|
134
|
-
config=self.config
|
135
|
-
): value
|
136
|
-
}
|
137
|
-
)
|
131
|
+
parent_config.request_option.inject_into_request(params, value, self.config)
|
138
132
|
return params
|
139
133
|
|
140
134
|
def stream_slices(self) -> Iterable[StreamSlice]:
|
@@ -295,58 +289,24 @@ class SubstreamPartitionRouter(PartitionRouter):
|
|
295
289
|
return
|
296
290
|
|
297
291
|
if not parent_state and incremental_dependency:
|
298
|
-
#
|
299
|
-
|
292
|
+
# Attempt to retrieve child state
|
293
|
+
substream_state = list(stream_state.values())
|
294
|
+
substream_state = substream_state[0] if substream_state else {} # type: ignore [assignment] # Incorrect type for assignment
|
295
|
+
parent_state = {}
|
296
|
+
|
297
|
+
# Copy child state to parent streams with incremental dependencies
|
298
|
+
if substream_state:
|
299
|
+
for parent_config in self.parent_stream_configs:
|
300
|
+
if parent_config.incremental_dependency:
|
301
|
+
parent_state[parent_config.stream.name] = {
|
302
|
+
parent_config.stream.cursor_field: substream_state
|
303
|
+
}
|
300
304
|
|
301
305
|
# Set state for each parent stream with an incremental dependency
|
302
306
|
for parent_config in self.parent_stream_configs:
|
303
307
|
if parent_config.incremental_dependency:
|
304
308
|
parent_config.stream.state = parent_state.get(parent_config.stream.name, {})
|
305
309
|
|
306
|
-
def _migrate_child_state_to_parent_state(self, stream_state: StreamState) -> StreamState:
|
307
|
-
"""
|
308
|
-
Migrate the child stream state to the parent stream's state format.
|
309
|
-
|
310
|
-
This method converts the global or child state into a format compatible with parent
|
311
|
-
streams. The migration occurs only for parent streams with incremental dependencies.
|
312
|
-
The method filters out per-partition states and retains only the global state in the
|
313
|
-
format `{cursor_field: cursor_value}`.
|
314
|
-
|
315
|
-
Args:
|
316
|
-
stream_state (StreamState): The state to migrate. Expected formats include:
|
317
|
-
- {"updated_at": "2023-05-27T00:00:00Z"}
|
318
|
-
- {"states": [...] } (ignored during migration)
|
319
|
-
|
320
|
-
Returns:
|
321
|
-
StreamState: A migrated state for parent streams in the format:
|
322
|
-
{
|
323
|
-
"parent_stream_name": {"parent_stream_cursor": "2023-05-27T00:00:00Z"}
|
324
|
-
}
|
325
|
-
|
326
|
-
Example:
|
327
|
-
Input: {"updated_at": "2023-05-27T00:00:00Z"}
|
328
|
-
Output: {
|
329
|
-
"parent_stream_name": {"parent_stream_cursor": "2023-05-27T00:00:00Z"}
|
330
|
-
}
|
331
|
-
"""
|
332
|
-
substream_state_values = list(stream_state.values())
|
333
|
-
substream_state = substream_state_values[0] if substream_state_values else {}
|
334
|
-
|
335
|
-
# Ignore per-partition states or invalid formats
|
336
|
-
if isinstance(substream_state, (list, dict)) or len(substream_state_values) != 1:
|
337
|
-
return {}
|
338
|
-
|
339
|
-
# Copy child state to parent streams with incremental dependencies
|
340
|
-
parent_state = {}
|
341
|
-
if substream_state:
|
342
|
-
for parent_config in self.parent_stream_configs:
|
343
|
-
if parent_config.incremental_dependency:
|
344
|
-
parent_state[parent_config.stream.name] = {
|
345
|
-
parent_config.stream.cursor_field: substream_state
|
346
|
-
}
|
347
|
-
|
348
|
-
return parent_state
|
349
|
-
|
350
310
|
def get_stream_state(self) -> Optional[Mapping[str, StreamState]]:
|
351
311
|
"""
|
352
312
|
Get the state of the parent streams.
|
@@ -8,7 +8,6 @@ from typing import Any, List, Mapping, Optional, Union
|
|
8
8
|
import requests
|
9
9
|
|
10
10
|
from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
|
11
|
-
from airbyte_cdk.sources.streams.http.error_handlers.backoff_strategy import BackoffStrategy
|
12
11
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
|
13
12
|
ErrorResolution,
|
14
13
|
ResponseAction,
|
@@ -78,24 +77,3 @@ class CompositeErrorHandler(ErrorHandler):
|
|
78
77
|
return matched_error_resolution
|
79
78
|
|
80
79
|
return create_fallback_error_resolution(response_or_exception)
|
81
|
-
|
82
|
-
@property
|
83
|
-
def backoff_strategies(self) -> Optional[List[BackoffStrategy]]:
|
84
|
-
"""
|
85
|
-
Combines backoff strategies from all child error handlers into a single flattened list.
|
86
|
-
|
87
|
-
When used with HttpRequester, note the following behavior:
|
88
|
-
- In HttpRequester.__post_init__, the entire list of backoff strategies is assigned to the error handler
|
89
|
-
- However, the error handler's backoff_time() method only ever uses the first non-None strategy in the list
|
90
|
-
- This means that if any backoff strategies are present, the first non-None strategy becomes the default
|
91
|
-
- This applies to both user-defined response filters and errors from DEFAULT_ERROR_MAPPING
|
92
|
-
- The list structure is not used to map different strategies to different error conditions
|
93
|
-
- Therefore, subsequent strategies in the list will not be used
|
94
|
-
|
95
|
-
Returns None if no handlers have strategies defined, which will result in HttpRequester using its default backoff strategy.
|
96
|
-
"""
|
97
|
-
all_strategies = []
|
98
|
-
for handler in self.error_handlers:
|
99
|
-
if hasattr(handler, "backoff_strategies") and handler.backoff_strategies:
|
100
|
-
all_strategies.extend(handler.backoff_strategies)
|
101
|
-
return all_strategies if all_strategies else None
|
@@ -151,16 +151,16 @@ class HttpResponseFilter:
|
|
151
151
|
:param response: The HTTP response which can be used during interpolation
|
152
152
|
:return: The evaluated error message string to be emitted
|
153
153
|
"""
|
154
|
-
return self.error_message.eval( # type: ignore[no-any-return, union-attr]
|
154
|
+
return self.error_message.eval( # type: ignore [no-any-return, union-attr]
|
155
155
|
self.config, response=self._safe_response_json(response), headers=response.headers
|
156
156
|
)
|
157
157
|
|
158
158
|
def _response_matches_predicate(self, response: requests.Response) -> bool:
|
159
159
|
return (
|
160
160
|
bool(
|
161
|
-
self.predicate.condition # type:ignore[union-attr]
|
162
|
-
and self.predicate.eval( # type:ignore[union-attr]
|
163
|
-
None, # type: ignore[arg-type]
|
161
|
+
self.predicate.condition # type: ignore [union-attr]
|
162
|
+
and self.predicate.eval( # type: ignore [union-attr]
|
163
|
+
None, # type: ignore [arg-type]
|
164
164
|
response=self._safe_response_json(response),
|
165
165
|
headers=response.headers,
|
166
166
|
)
|
@@ -199,6 +199,9 @@ class HttpRequester(Requester):
|
|
199
199
|
Raise a ValueError if there's a key collision
|
200
200
|
Returned merged mapping otherwise
|
201
201
|
"""
|
202
|
+
|
203
|
+
is_body_json = requester_method.__name__ == "get_request_body_json"
|
204
|
+
|
202
205
|
return combine_mappings(
|
203
206
|
[
|
204
207
|
requester_method(
|
@@ -208,7 +211,8 @@ class HttpRequester(Requester):
|
|
208
211
|
),
|
209
212
|
auth_options_method(),
|
210
213
|
extra_options,
|
211
|
-
]
|
214
|
+
],
|
215
|
+
allow_same_value_merge=is_body_json,
|
212
216
|
)
|
213
217
|
|
214
218
|
def _request_headers(
|
@@ -187,7 +187,7 @@ class DefaultPaginator(Paginator):
|
|
187
187
|
def _get_request_options(
|
188
188
|
self, option_type: RequestOptionType, next_page_token: Optional[Mapping[str, Any]]
|
189
189
|
) -> MutableMapping[str, Any]:
|
190
|
-
options = {}
|
190
|
+
options: MutableMapping[str, Any] = {}
|
191
191
|
|
192
192
|
token = next_page_token.get("next_page_token") if next_page_token else None
|
193
193
|
if (
|
@@ -196,15 +196,16 @@ class DefaultPaginator(Paginator):
|
|
196
196
|
and isinstance(self.page_token_option, RequestOption)
|
197
197
|
and self.page_token_option.inject_into == option_type
|
198
198
|
):
|
199
|
-
|
199
|
+
self.page_token_option.inject_into_request(options, token, self.config)
|
200
|
+
|
200
201
|
if (
|
201
202
|
self.page_size_option
|
202
203
|
and self.pagination_strategy.get_page_size()
|
203
204
|
and self.page_size_option.inject_into == option_type
|
204
205
|
):
|
205
|
-
|
206
|
-
|
207
|
-
|
206
|
+
page_size = self.pagination_strategy.get_page_size()
|
207
|
+
self.page_size_option.inject_into_request(options, page_size, self.config)
|
208
|
+
|
208
209
|
return options
|
209
210
|
|
210
211
|
|
@@ -4,9 +4,10 @@
|
|
4
4
|
|
5
5
|
from dataclasses import InitVar, dataclass
|
6
6
|
from enum import Enum
|
7
|
-
from typing import Any, Mapping, Union
|
7
|
+
from typing import Any, List, Literal, Mapping, MutableMapping, Optional, Union
|
8
8
|
|
9
9
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
10
|
+
from airbyte_cdk.sources.types import Config
|
10
11
|
|
11
12
|
|
12
13
|
class RequestOptionType(Enum):
|
@@ -26,13 +27,91 @@ class RequestOption:
|
|
26
27
|
Describes an option to set on a request
|
27
28
|
|
28
29
|
Attributes:
|
29
|
-
field_name (str): Describes the name of the parameter to inject
|
30
|
+
field_name (str): Describes the name of the parameter to inject. Mutually exclusive with field_path.
|
31
|
+
field_path (list(str)): Describes the path to a nested field as a list of field names.
|
32
|
+
Only valid for body_json injection type, and mutually exclusive with field_name.
|
30
33
|
inject_into (RequestOptionType): Describes where in the HTTP request to inject the parameter
|
31
34
|
"""
|
32
35
|
|
33
|
-
field_name: Union[InterpolatedString, str]
|
34
36
|
inject_into: RequestOptionType
|
35
37
|
parameters: InitVar[Mapping[str, Any]]
|
38
|
+
field_name: Optional[Union[InterpolatedString, str]] = None
|
39
|
+
field_path: Optional[List[Union[InterpolatedString, str]]] = None
|
36
40
|
|
37
41
|
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
38
|
-
|
42
|
+
# Validate inputs. We should expect either field_name or field_path, but not both
|
43
|
+
if self.field_name is None and self.field_path is None:
|
44
|
+
raise ValueError("RequestOption requires either a field_name or field_path")
|
45
|
+
|
46
|
+
if self.field_name is not None and self.field_path is not None:
|
47
|
+
raise ValueError(
|
48
|
+
"Only one of field_name or field_path can be provided to RequestOption"
|
49
|
+
)
|
50
|
+
|
51
|
+
# Nested field injection is only supported for body JSON injection
|
52
|
+
if self.field_path is not None and self.inject_into != RequestOptionType.body_json:
|
53
|
+
raise ValueError(
|
54
|
+
"Nested field injection is only supported for body JSON injection. Please use a top-level field_name for other injection types."
|
55
|
+
)
|
56
|
+
|
57
|
+
# Convert field_name and field_path into InterpolatedString objects if they are strings
|
58
|
+
if self.field_name is not None:
|
59
|
+
self.field_name = InterpolatedString.create(self.field_name, parameters=parameters)
|
60
|
+
elif self.field_path is not None:
|
61
|
+
self.field_path = [
|
62
|
+
InterpolatedString.create(segment, parameters=parameters)
|
63
|
+
for segment in self.field_path
|
64
|
+
]
|
65
|
+
|
66
|
+
@property
|
67
|
+
def _is_field_path(self) -> bool:
|
68
|
+
"""Returns whether this option is a field path (ie, a nested field)"""
|
69
|
+
return self.field_path is not None
|
70
|
+
|
71
|
+
def inject_into_request(
|
72
|
+
self,
|
73
|
+
target: MutableMapping[str, Any],
|
74
|
+
value: Any,
|
75
|
+
config: Config,
|
76
|
+
) -> None:
|
77
|
+
"""
|
78
|
+
Inject a request option value into a target request structure using either field_name or field_path.
|
79
|
+
For non-body-json injection, only top-level field names are supported.
|
80
|
+
For body-json injection, both field names and nested field paths are supported.
|
81
|
+
|
82
|
+
Args:
|
83
|
+
target: The request structure to inject the value into
|
84
|
+
value: The value to inject
|
85
|
+
config: The config object to use for interpolation
|
86
|
+
"""
|
87
|
+
if self._is_field_path:
|
88
|
+
if self.inject_into != RequestOptionType.body_json:
|
89
|
+
raise ValueError(
|
90
|
+
"Nested field injection is only supported for body JSON injection. Please use a top-level field_name for other injection types."
|
91
|
+
)
|
92
|
+
|
93
|
+
assert self.field_path is not None # for type checker
|
94
|
+
current = target
|
95
|
+
# Convert path segments into strings, evaluating any interpolated segments
|
96
|
+
# Example: ["data", "{{ config[user_type] }}", "id"] -> ["data", "admin", "id"]
|
97
|
+
*path_parts, final_key = [
|
98
|
+
str(
|
99
|
+
segment.eval(config=config)
|
100
|
+
if isinstance(segment, InterpolatedString)
|
101
|
+
else segment
|
102
|
+
)
|
103
|
+
for segment in self.field_path
|
104
|
+
]
|
105
|
+
|
106
|
+
# Build a nested dictionary structure and set the final value at the deepest level
|
107
|
+
for part in path_parts:
|
108
|
+
current = current.setdefault(part, {})
|
109
|
+
current[final_key] = value
|
110
|
+
else:
|
111
|
+
# For non-nested fields, evaluate the field name if it's an interpolated string
|
112
|
+
key = (
|
113
|
+
self.field_name.eval(config=config)
|
114
|
+
if isinstance(self.field_name, InterpolatedString)
|
115
|
+
else self.field_name
|
116
|
+
)
|
117
|
+
target[str(key)] = value
|
@@ -80,12 +80,13 @@ class DatetimeBasedRequestOptionsProvider(RequestOptionsProvider):
|
|
80
80
|
options: MutableMapping[str, Any] = {}
|
81
81
|
if not stream_slice:
|
82
82
|
return options
|
83
|
+
|
83
84
|
if self.start_time_option and self.start_time_option.inject_into == option_type:
|
84
|
-
|
85
|
-
|
86
|
-
|
85
|
+
start_time_value = stream_slice.get(self._partition_field_start.eval(self.config))
|
86
|
+
self.start_time_option.inject_into_request(options, start_time_value, self.config)
|
87
|
+
|
87
88
|
if self.end_time_option and self.end_time_option.inject_into == option_type:
|
88
|
-
|
89
|
-
|
90
|
-
|
89
|
+
end_time_value = stream_slice.get(self._partition_field_end.eval(self.config))
|
90
|
+
self.end_time_option.inject_into_request(options, end_time_value, self.config)
|
91
|
+
|
91
92
|
return options
|
@@ -6,7 +6,7 @@ from typing import Any, Iterable, Mapping, Optional
|
|
6
6
|
|
7
7
|
from typing_extensions import deprecated
|
8
8
|
|
9
|
-
from airbyte_cdk.
|
9
|
+
from airbyte_cdk.models import FailureType
|
10
10
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncPartition
|
11
11
|
from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
|
12
12
|
from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
|
@@ -16,6 +16,7 @@ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
|
|
16
16
|
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
17
17
|
from airbyte_cdk.sources.streams.core import StreamData
|
18
18
|
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
19
|
+
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
19
20
|
|
20
21
|
|
21
22
|
@deprecated(
|
@@ -56,9 +57,9 @@ class AsyncRetriever(Retriever):
|
|
56
57
|
|
57
58
|
return self.state
|
58
59
|
|
59
|
-
def
|
60
|
+
def _validate_and_get_stream_slice_partition(
|
60
61
|
self, stream_slice: Optional[StreamSlice] = None
|
61
|
-
) ->
|
62
|
+
) -> AsyncPartition:
|
62
63
|
"""
|
63
64
|
Validates the stream_slice argument and returns the partition from it.
|
64
65
|
|
@@ -72,7 +73,12 @@ class AsyncRetriever(Retriever):
|
|
72
73
|
AirbyteTracedException: If the stream_slice is not an instance of StreamSlice or if the partition is not present in the stream_slice.
|
73
74
|
|
74
75
|
"""
|
75
|
-
|
76
|
+
if not isinstance(stream_slice, StreamSlice) or "partition" not in stream_slice.partition:
|
77
|
+
raise AirbyteTracedException(
|
78
|
+
message="Invalid arguments to AsyncJobRetriever.read_records: stream_slice is no optional. Please contact Airbyte Support",
|
79
|
+
failure_type=FailureType.system_error,
|
80
|
+
)
|
81
|
+
return stream_slice["partition"] # type: ignore # stream_slice["partition"] has been added as an AsyncPartition as part of stream_slices
|
76
82
|
|
77
83
|
def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
|
78
84
|
return self.stream_slicer.stream_slices()
|
@@ -83,8 +89,8 @@ class AsyncRetriever(Retriever):
|
|
83
89
|
stream_slice: Optional[StreamSlice] = None,
|
84
90
|
) -> Iterable[StreamData]:
|
85
91
|
stream_state: StreamState = self._get_stream_state()
|
86
|
-
|
87
|
-
records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(
|
92
|
+
partition: AsyncPartition = self._validate_and_get_stream_slice_partition(stream_slice)
|
93
|
+
records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(partition)
|
88
94
|
|
89
95
|
yield from self.record_selector.filter_and_transform(
|
90
96
|
all_data=records,
|
@@ -128,6 +128,9 @@ class SimpleRetriever(Retriever):
|
|
128
128
|
Returned merged mapping otherwise
|
129
129
|
"""
|
130
130
|
# FIXME we should eventually remove the usage of stream_state as part of the interpolation
|
131
|
+
|
132
|
+
is_body_json = paginator_method.__name__ == "get_request_body_json"
|
133
|
+
|
131
134
|
mappings = [
|
132
135
|
paginator_method(
|
133
136
|
stream_state=stream_state,
|
@@ -143,7 +146,7 @@ class SimpleRetriever(Retriever):
|
|
143
146
|
next_page_token=next_page_token,
|
144
147
|
)
|
145
148
|
)
|
146
|
-
return combine_mappings(mappings)
|
149
|
+
return combine_mappings(mappings, allow_same_value_merge=is_body_json)
|
147
150
|
|
148
151
|
def _request_headers(
|
149
152
|
self,
|
@@ -160,7 +163,7 @@ class SimpleRetriever(Retriever):
|
|
160
163
|
stream_slice,
|
161
164
|
next_page_token,
|
162
165
|
self._paginator.get_request_headers,
|
163
|
-
self.
|
166
|
+
self.stream_slicer.get_request_headers,
|
164
167
|
)
|
165
168
|
if isinstance(headers, str):
|
166
169
|
raise ValueError("Request headers cannot be a string")
|
@@ -4,7 +4,6 @@
|
|
4
4
|
|
5
5
|
from airbyte_cdk.sources.declarative.schema.default_schema_loader import DefaultSchemaLoader
|
6
6
|
from airbyte_cdk.sources.declarative.schema.dynamic_schema_loader import (
|
7
|
-
ComplexFieldType,
|
8
7
|
DynamicSchemaLoader,
|
9
8
|
SchemaTypeIdentifier,
|
10
9
|
TypesMap,
|
@@ -19,7 +18,6 @@ __all__ = [
|
|
19
18
|
"SchemaLoader",
|
20
19
|
"InlineSchemaLoader",
|
21
20
|
"DynamicSchemaLoader",
|
22
|
-
"ComplexFieldType",
|
23
21
|
"TypesMap",
|
24
22
|
"SchemaTypeIdentifier",
|
25
23
|
]
|
@@ -18,7 +18,7 @@ from airbyte_cdk.sources.declarative.transformations import RecordTransformation
|
|
18
18
|
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
19
19
|
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
20
20
|
|
21
|
-
AIRBYTE_DATA_TYPES: Mapping[str,
|
21
|
+
AIRBYTE_DATA_TYPES: Mapping[str, Mapping[str, Any]] = {
|
22
22
|
"string": {"type": ["null", "string"]},
|
23
23
|
"boolean": {"type": ["null", "boolean"]},
|
24
24
|
"date": {"type": ["null", "string"], "format": "date"},
|
@@ -45,25 +45,6 @@ AIRBYTE_DATA_TYPES: Mapping[str, MutableMapping[str, Any]] = {
|
|
45
45
|
}
|
46
46
|
|
47
47
|
|
48
|
-
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
49
|
-
@dataclass(frozen=True)
|
50
|
-
class ComplexFieldType:
|
51
|
-
"""
|
52
|
-
Identifies complex field type
|
53
|
-
"""
|
54
|
-
|
55
|
-
field_type: str
|
56
|
-
items: Optional[Union[str, "ComplexFieldType"]] = None
|
57
|
-
|
58
|
-
def __post_init__(self) -> None:
|
59
|
-
"""
|
60
|
-
Enforces that `items` is only used when `field_type` is a array
|
61
|
-
"""
|
62
|
-
# `items_type` is valid only for array target types
|
63
|
-
if self.items and self.field_type != "array":
|
64
|
-
raise ValueError("'items' can only be used when 'field_type' is an array.")
|
65
|
-
|
66
|
-
|
67
48
|
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
68
49
|
@dataclass(frozen=True)
|
69
50
|
class TypesMap:
|
@@ -71,7 +52,7 @@ class TypesMap:
|
|
71
52
|
Represents a mapping between a current type and its corresponding target type.
|
72
53
|
"""
|
73
54
|
|
74
|
-
target_type: Union[List[str], str
|
55
|
+
target_type: Union[List[str], str]
|
75
56
|
current_type: Union[List[str], str]
|
76
57
|
condition: Optional[str]
|
77
58
|
|
@@ -154,9 +135,8 @@ class DynamicSchemaLoader(SchemaLoader):
|
|
154
135
|
transformed_properties = self._transform(properties, {})
|
155
136
|
|
156
137
|
return {
|
157
|
-
"$schema": "
|
138
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
158
139
|
"type": "object",
|
159
|
-
"additionalProperties": True,
|
160
140
|
"properties": transformed_properties,
|
161
141
|
}
|
162
142
|
|
@@ -208,37 +188,18 @@ class DynamicSchemaLoader(SchemaLoader):
|
|
208
188
|
first_type = self._get_airbyte_type(mapped_field_type[0])
|
209
189
|
second_type = self._get_airbyte_type(mapped_field_type[1])
|
210
190
|
return {"oneOf": [first_type, second_type]}
|
211
|
-
|
212
191
|
elif isinstance(mapped_field_type, str):
|
213
192
|
return self._get_airbyte_type(mapped_field_type)
|
214
|
-
|
215
|
-
elif isinstance(mapped_field_type, ComplexFieldType):
|
216
|
-
return self._resolve_complex_type(mapped_field_type)
|
217
|
-
|
218
193
|
else:
|
219
194
|
raise ValueError(
|
220
195
|
f"Invalid data type. Available string or two items list of string. Got {mapped_field_type}."
|
221
196
|
)
|
222
197
|
|
223
|
-
def _resolve_complex_type(self, complex_type: ComplexFieldType) -> Mapping[str, Any]:
|
224
|
-
if not complex_type.items:
|
225
|
-
return self._get_airbyte_type(complex_type.field_type)
|
226
|
-
|
227
|
-
field_type = self._get_airbyte_type(complex_type.field_type)
|
228
|
-
|
229
|
-
field_type["items"] = (
|
230
|
-
self._get_airbyte_type(complex_type.items)
|
231
|
-
if isinstance(complex_type.items, str)
|
232
|
-
else self._resolve_complex_type(complex_type.items)
|
233
|
-
)
|
234
|
-
|
235
|
-
return field_type
|
236
|
-
|
237
198
|
def _replace_type_if_not_valid(
|
238
199
|
self,
|
239
200
|
field_type: Union[List[str], str],
|
240
201
|
raw_schema: MutableMapping[str, Any],
|
241
|
-
) -> Union[List[str], str
|
202
|
+
) -> Union[List[str], str]:
|
242
203
|
"""
|
243
204
|
Replaces a field type if it matches a type mapping in `types_map`.
|
244
205
|
"""
|
@@ -255,7 +216,7 @@ class DynamicSchemaLoader(SchemaLoader):
|
|
255
216
|
return field_type
|
256
217
|
|
257
218
|
@staticmethod
|
258
|
-
def _get_airbyte_type(field_type: str) ->
|
219
|
+
def _get_airbyte_type(field_type: str) -> Mapping[str, Any]:
|
259
220
|
"""
|
260
221
|
Maps a field type to its corresponding Airbyte type definition.
|
261
222
|
"""
|
@@ -45,7 +45,7 @@ def format_http_message(
|
|
45
45
|
log_message["http"]["is_auxiliary"] = is_auxiliary # type: ignore [index]
|
46
46
|
if stream_name:
|
47
47
|
log_message["airbyte_cdk"] = {"stream": {"name": stream_name}}
|
48
|
-
return log_message # type: ignore[return-value] # got "dict[str, object]", expected "dict[str, JsonType]"
|
48
|
+
return log_message # type: ignore [return-value] # got "dict[str, object]", expected "dict[str, JsonType]"
|
49
49
|
|
50
50
|
|
51
51
|
def _normalize_body_string(body_str: Optional[Union[str, bytes]]) -> Optional[str]:
|