airbyte-cdk 6.31.2.dev0__py3-none-any.whl → 6.32.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +9 -3
- airbyte_cdk/connector_builder/connector_builder_handler.py +3 -2
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +7 -7
- airbyte_cdk/sources/declarative/auth/jwt.py +17 -11
- airbyte_cdk/sources/declarative/auth/oauth.py +89 -23
- airbyte_cdk/sources/declarative/auth/token.py +8 -3
- airbyte_cdk/sources/declarative/auth/token_provider.py +4 -5
- airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +19 -9
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +134 -43
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +55 -16
- airbyte_cdk/sources/declarative/declarative_stream.py +3 -1
- airbyte_cdk/sources/declarative/extractors/record_filter.py +3 -5
- airbyte_cdk/sources/declarative/incremental/__init__.py +6 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +400 -0
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +6 -7
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +3 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +35 -3
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +20 -7
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +45 -15
- airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +143 -0
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +343 -64
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +5 -5
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +2 -4
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +55 -15
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +22 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +4 -4
- airbyte_cdk/sources/declarative/requesters/http_requester.py +1 -5
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +5 -6
- airbyte_cdk/sources/declarative/requesters/request_option.py +4 -83
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +6 -7
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +6 -12
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +2 -5
- airbyte_cdk/sources/declarative/schema/__init__.py +2 -0
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +44 -5
- airbyte_cdk/sources/http_logger.py +1 -1
- airbyte_cdk/sources/streams/concurrent/clamping.py +99 -0
- airbyte_cdk/sources/streams/concurrent/cursor.py +51 -57
- airbyte_cdk/sources/streams/concurrent/cursor_types.py +32 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +22 -13
- airbyte_cdk/sources/streams/core.py +6 -6
- airbyte_cdk/sources/streams/http/http.py +1 -2
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +231 -62
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +171 -88
- airbyte_cdk/sources/types.py +4 -2
- airbyte_cdk/sources/utils/transform.py +23 -2
- airbyte_cdk/test/utils/manifest_only_fixtures.py +1 -2
- airbyte_cdk/utils/datetime_helpers.py +499 -0
- airbyte_cdk/utils/mapping_helpers.py +27 -86
- airbyte_cdk/utils/slice_hasher.py +8 -1
- airbyte_cdk-6.32.0.dist-info/LICENSE_SHORT +1 -0
- {airbyte_cdk-6.31.2.dev0.dist-info → airbyte_cdk-6.32.0.dist-info}/METADATA +6 -6
- {airbyte_cdk-6.31.2.dev0.dist-info → airbyte_cdk-6.32.0.dist-info}/RECORD +55 -49
- {airbyte_cdk-6.31.2.dev0.dist-info → airbyte_cdk-6.32.0.dist-info}/WHEEL +1 -1
- {airbyte_cdk-6.31.2.dev0.dist-info → airbyte_cdk-6.32.0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.31.2.dev0.dist-info → airbyte_cdk-6.32.0.dist-info}/entry_points.txt +0 -0
@@ -4,9 +4,9 @@ from dataclasses import InitVar, dataclass, field
|
|
4
4
|
from typing import Any, Callable, Iterable, Mapping, Optional
|
5
5
|
|
6
6
|
from airbyte_cdk.models import FailureType
|
7
|
+
from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
|
7
8
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import (
|
8
9
|
AsyncJobOrchestrator,
|
9
|
-
AsyncPartition,
|
10
10
|
)
|
11
11
|
from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import (
|
12
12
|
SinglePartitionRouter,
|
@@ -42,12 +42,12 @@ class AsyncJobPartitionRouter(StreamSlicer):
|
|
42
42
|
|
43
43
|
for completed_partition in self._job_orchestrator.create_and_get_completed_partitions():
|
44
44
|
yield StreamSlice(
|
45
|
-
partition=dict(completed_partition.stream_slice.partition)
|
46
|
-
| {"partition": completed_partition},
|
45
|
+
partition=dict(completed_partition.stream_slice.partition),
|
47
46
|
cursor_slice=completed_partition.stream_slice.cursor_slice,
|
47
|
+
extra_fields={"jobs": list(completed_partition.jobs)},
|
48
48
|
)
|
49
49
|
|
50
|
-
def fetch_records(self,
|
50
|
+
def fetch_records(self, async_jobs: Iterable[AsyncJob]) -> Iterable[Mapping[str, Any]]:
|
51
51
|
"""
|
52
52
|
This method of fetching records extends beyond what a PartitionRouter/StreamSlicer should
|
53
53
|
be responsible for. However, this was added in because the JobOrchestrator is required to
|
@@ -62,4 +62,4 @@ class AsyncJobPartitionRouter(StreamSlicer):
|
|
62
62
|
failure_type=FailureType.system_error,
|
63
63
|
)
|
64
64
|
|
65
|
-
return self._job_orchestrator.fetch_records(
|
65
|
+
return self._job_orchestrator.fetch_records(async_jobs=async_jobs)
|
@@ -3,7 +3,7 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
from dataclasses import InitVar, dataclass
|
6
|
-
from typing import Any, Iterable, List, Mapping,
|
6
|
+
from typing import Any, Iterable, List, Mapping, Optional, Union
|
7
7
|
|
8
8
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
9
9
|
from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
|
@@ -100,9 +100,7 @@ class ListPartitionRouter(PartitionRouter):
|
|
100
100
|
):
|
101
101
|
slice_value = stream_slice.get(self._cursor_field.eval(self.config))
|
102
102
|
if slice_value:
|
103
|
-
|
104
|
-
self.request_option.inject_into_request(options, slice_value, self.config)
|
105
|
-
return options
|
103
|
+
return {self.request_option.field_name.eval(self.config): slice_value} # type: ignore # field_name is always casted to InterpolatedString
|
106
104
|
else:
|
107
105
|
return {}
|
108
106
|
else:
|
@@ -4,7 +4,7 @@
|
|
4
4
|
import copy
|
5
5
|
import logging
|
6
6
|
from dataclasses import InitVar, dataclass
|
7
|
-
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping,
|
7
|
+
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import dpath
|
10
10
|
|
@@ -118,7 +118,7 @@ class SubstreamPartitionRouter(PartitionRouter):
|
|
118
118
|
def _get_request_option(
|
119
119
|
self, option_type: RequestOptionType, stream_slice: Optional[StreamSlice]
|
120
120
|
) -> Mapping[str, Any]:
|
121
|
-
params
|
121
|
+
params = {}
|
122
122
|
if stream_slice:
|
123
123
|
for parent_config in self.parent_stream_configs:
|
124
124
|
if (
|
@@ -128,7 +128,13 @@ class SubstreamPartitionRouter(PartitionRouter):
|
|
128
128
|
key = parent_config.partition_field.eval(self.config) # type: ignore # partition_field is always casted to an interpolated string
|
129
129
|
value = stream_slice.get(key)
|
130
130
|
if value:
|
131
|
-
|
131
|
+
params.update(
|
132
|
+
{
|
133
|
+
parent_config.request_option.field_name.eval( # type: ignore [union-attr]
|
134
|
+
config=self.config
|
135
|
+
): value
|
136
|
+
}
|
137
|
+
)
|
132
138
|
return params
|
133
139
|
|
134
140
|
def stream_slices(self) -> Iterable[StreamSlice]:
|
@@ -289,24 +295,58 @@ class SubstreamPartitionRouter(PartitionRouter):
|
|
289
295
|
return
|
290
296
|
|
291
297
|
if not parent_state and incremental_dependency:
|
292
|
-
#
|
293
|
-
|
294
|
-
substream_state = substream_state[0] if substream_state else {} # type: ignore [assignment] # Incorrect type for assignment
|
295
|
-
parent_state = {}
|
296
|
-
|
297
|
-
# Copy child state to parent streams with incremental dependencies
|
298
|
-
if substream_state:
|
299
|
-
for parent_config in self.parent_stream_configs:
|
300
|
-
if parent_config.incremental_dependency:
|
301
|
-
parent_state[parent_config.stream.name] = {
|
302
|
-
parent_config.stream.cursor_field: substream_state
|
303
|
-
}
|
298
|
+
# Migrate child state to parent state format
|
299
|
+
parent_state = self._migrate_child_state_to_parent_state(stream_state)
|
304
300
|
|
305
301
|
# Set state for each parent stream with an incremental dependency
|
306
302
|
for parent_config in self.parent_stream_configs:
|
307
303
|
if parent_config.incremental_dependency:
|
308
304
|
parent_config.stream.state = parent_state.get(parent_config.stream.name, {})
|
309
305
|
|
306
|
+
def _migrate_child_state_to_parent_state(self, stream_state: StreamState) -> StreamState:
|
307
|
+
"""
|
308
|
+
Migrate the child stream state to the parent stream's state format.
|
309
|
+
|
310
|
+
This method converts the global or child state into a format compatible with parent
|
311
|
+
streams. The migration occurs only for parent streams with incremental dependencies.
|
312
|
+
The method filters out per-partition states and retains only the global state in the
|
313
|
+
format `{cursor_field: cursor_value}`.
|
314
|
+
|
315
|
+
Args:
|
316
|
+
stream_state (StreamState): The state to migrate. Expected formats include:
|
317
|
+
- {"updated_at": "2023-05-27T00:00:00Z"}
|
318
|
+
- {"states": [...] } (ignored during migration)
|
319
|
+
|
320
|
+
Returns:
|
321
|
+
StreamState: A migrated state for parent streams in the format:
|
322
|
+
{
|
323
|
+
"parent_stream_name": {"parent_stream_cursor": "2023-05-27T00:00:00Z"}
|
324
|
+
}
|
325
|
+
|
326
|
+
Example:
|
327
|
+
Input: {"updated_at": "2023-05-27T00:00:00Z"}
|
328
|
+
Output: {
|
329
|
+
"parent_stream_name": {"parent_stream_cursor": "2023-05-27T00:00:00Z"}
|
330
|
+
}
|
331
|
+
"""
|
332
|
+
substream_state_values = list(stream_state.values())
|
333
|
+
substream_state = substream_state_values[0] if substream_state_values else {}
|
334
|
+
|
335
|
+
# Ignore per-partition states or invalid formats
|
336
|
+
if isinstance(substream_state, (list, dict)) or len(substream_state_values) != 1:
|
337
|
+
return {}
|
338
|
+
|
339
|
+
# Copy child state to parent streams with incremental dependencies
|
340
|
+
parent_state = {}
|
341
|
+
if substream_state:
|
342
|
+
for parent_config in self.parent_stream_configs:
|
343
|
+
if parent_config.incremental_dependency:
|
344
|
+
parent_state[parent_config.stream.name] = {
|
345
|
+
parent_config.stream.cursor_field: substream_state
|
346
|
+
}
|
347
|
+
|
348
|
+
return parent_state
|
349
|
+
|
310
350
|
def get_stream_state(self) -> Optional[Mapping[str, StreamState]]:
|
311
351
|
"""
|
312
352
|
Get the state of the parent streams.
|
@@ -8,6 +8,7 @@ from typing import Any, List, Mapping, Optional, Union
|
|
8
8
|
import requests
|
9
9
|
|
10
10
|
from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
|
11
|
+
from airbyte_cdk.sources.streams.http.error_handlers.backoff_strategy import BackoffStrategy
|
11
12
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
|
12
13
|
ErrorResolution,
|
13
14
|
ResponseAction,
|
@@ -77,3 +78,24 @@ class CompositeErrorHandler(ErrorHandler):
|
|
77
78
|
return matched_error_resolution
|
78
79
|
|
79
80
|
return create_fallback_error_resolution(response_or_exception)
|
81
|
+
|
82
|
+
@property
|
83
|
+
def backoff_strategies(self) -> Optional[List[BackoffStrategy]]:
|
84
|
+
"""
|
85
|
+
Combines backoff strategies from all child error handlers into a single flattened list.
|
86
|
+
|
87
|
+
When used with HttpRequester, note the following behavior:
|
88
|
+
- In HttpRequester.__post_init__, the entire list of backoff strategies is assigned to the error handler
|
89
|
+
- However, the error handler's backoff_time() method only ever uses the first non-None strategy in the list
|
90
|
+
- This means that if any backoff strategies are present, the first non-None strategy becomes the default
|
91
|
+
- This applies to both user-defined response filters and errors from DEFAULT_ERROR_MAPPING
|
92
|
+
- The list structure is not used to map different strategies to different error conditions
|
93
|
+
- Therefore, subsequent strategies in the list will not be used
|
94
|
+
|
95
|
+
Returns None if no handlers have strategies defined, which will result in HttpRequester using its default backoff strategy.
|
96
|
+
"""
|
97
|
+
all_strategies = []
|
98
|
+
for handler in self.error_handlers:
|
99
|
+
if hasattr(handler, "backoff_strategies") and handler.backoff_strategies:
|
100
|
+
all_strategies.extend(handler.backoff_strategies)
|
101
|
+
return all_strategies if all_strategies else None
|
@@ -151,16 +151,16 @@ class HttpResponseFilter:
|
|
151
151
|
:param response: The HTTP response which can be used during interpolation
|
152
152
|
:return: The evaluated error message string to be emitted
|
153
153
|
"""
|
154
|
-
return self.error_message.eval( # type: ignore
|
154
|
+
return self.error_message.eval( # type: ignore[no-any-return, union-attr]
|
155
155
|
self.config, response=self._safe_response_json(response), headers=response.headers
|
156
156
|
)
|
157
157
|
|
158
158
|
def _response_matches_predicate(self, response: requests.Response) -> bool:
|
159
159
|
return (
|
160
160
|
bool(
|
161
|
-
self.predicate.condition # type:
|
162
|
-
and self.predicate.eval( # type:
|
163
|
-
None, # type: ignore
|
161
|
+
self.predicate.condition # type:ignore[union-attr]
|
162
|
+
and self.predicate.eval( # type:ignore[union-attr]
|
163
|
+
None, # type: ignore[arg-type]
|
164
164
|
response=self._safe_response_json(response),
|
165
165
|
headers=response.headers,
|
166
166
|
)
|
@@ -199,9 +199,6 @@ class HttpRequester(Requester):
|
|
199
199
|
Raise a ValueError if there's a key collision
|
200
200
|
Returned merged mapping otherwise
|
201
201
|
"""
|
202
|
-
|
203
|
-
is_body_json = requester_method.__name__ == "get_request_body_json"
|
204
|
-
|
205
202
|
return combine_mappings(
|
206
203
|
[
|
207
204
|
requester_method(
|
@@ -211,8 +208,7 @@ class HttpRequester(Requester):
|
|
211
208
|
),
|
212
209
|
auth_options_method(),
|
213
210
|
extra_options,
|
214
|
-
]
|
215
|
-
allow_same_value_merge=is_body_json,
|
211
|
+
]
|
216
212
|
)
|
217
213
|
|
218
214
|
def _request_headers(
|
@@ -187,7 +187,7 @@ class DefaultPaginator(Paginator):
|
|
187
187
|
def _get_request_options(
|
188
188
|
self, option_type: RequestOptionType, next_page_token: Optional[Mapping[str, Any]]
|
189
189
|
) -> MutableMapping[str, Any]:
|
190
|
-
options
|
190
|
+
options = {}
|
191
191
|
|
192
192
|
token = next_page_token.get("next_page_token") if next_page_token else None
|
193
193
|
if (
|
@@ -196,16 +196,15 @@ class DefaultPaginator(Paginator):
|
|
196
196
|
and isinstance(self.page_token_option, RequestOption)
|
197
197
|
and self.page_token_option.inject_into == option_type
|
198
198
|
):
|
199
|
-
self.page_token_option.
|
200
|
-
|
199
|
+
options[self.page_token_option.field_name.eval(config=self.config)] = token # type: ignore # field_name is always cast to an interpolated string
|
201
200
|
if (
|
202
201
|
self.page_size_option
|
203
202
|
and self.pagination_strategy.get_page_size()
|
204
203
|
and self.page_size_option.inject_into == option_type
|
205
204
|
):
|
206
|
-
|
207
|
-
|
208
|
-
|
205
|
+
options[self.page_size_option.field_name.eval(config=self.config)] = ( # type: ignore [union-attr]
|
206
|
+
self.pagination_strategy.get_page_size()
|
207
|
+
) # type: ignore # field_name is always cast to an interpolated string
|
209
208
|
return options
|
210
209
|
|
211
210
|
|
@@ -4,10 +4,9 @@
|
|
4
4
|
|
5
5
|
from dataclasses import InitVar, dataclass
|
6
6
|
from enum import Enum
|
7
|
-
from typing import Any,
|
7
|
+
from typing import Any, Mapping, Union
|
8
8
|
|
9
9
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
10
|
-
from airbyte_cdk.sources.types import Config
|
11
10
|
|
12
11
|
|
13
12
|
class RequestOptionType(Enum):
|
@@ -27,91 +26,13 @@ class RequestOption:
|
|
27
26
|
Describes an option to set on a request
|
28
27
|
|
29
28
|
Attributes:
|
30
|
-
field_name (str): Describes the name of the parameter to inject
|
31
|
-
field_path (list(str)): Describes the path to a nested field as a list of field names.
|
32
|
-
Only valid for body_json injection type, and mutually exclusive with field_name.
|
29
|
+
field_name (str): Describes the name of the parameter to inject
|
33
30
|
inject_into (RequestOptionType): Describes where in the HTTP request to inject the parameter
|
34
31
|
"""
|
35
32
|
|
33
|
+
field_name: Union[InterpolatedString, str]
|
36
34
|
inject_into: RequestOptionType
|
37
35
|
parameters: InitVar[Mapping[str, Any]]
|
38
|
-
field_name: Optional[Union[InterpolatedString, str]] = None
|
39
|
-
field_path: Optional[List[Union[InterpolatedString, str]]] = None
|
40
36
|
|
41
37
|
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
42
|
-
|
43
|
-
if self.field_name is None and self.field_path is None:
|
44
|
-
raise ValueError("RequestOption requires either a field_name or field_path")
|
45
|
-
|
46
|
-
if self.field_name is not None and self.field_path is not None:
|
47
|
-
raise ValueError(
|
48
|
-
"Only one of field_name or field_path can be provided to RequestOption"
|
49
|
-
)
|
50
|
-
|
51
|
-
# Nested field injection is only supported for body JSON injection
|
52
|
-
if self.field_path is not None and self.inject_into != RequestOptionType.body_json:
|
53
|
-
raise ValueError(
|
54
|
-
"Nested field injection is only supported for body JSON injection. Please use a top-level field_name for other injection types."
|
55
|
-
)
|
56
|
-
|
57
|
-
# Convert field_name and field_path into InterpolatedString objects if they are strings
|
58
|
-
if self.field_name is not None:
|
59
|
-
self.field_name = InterpolatedString.create(self.field_name, parameters=parameters)
|
60
|
-
elif self.field_path is not None:
|
61
|
-
self.field_path = [
|
62
|
-
InterpolatedString.create(segment, parameters=parameters)
|
63
|
-
for segment in self.field_path
|
64
|
-
]
|
65
|
-
|
66
|
-
@property
|
67
|
-
def _is_field_path(self) -> bool:
|
68
|
-
"""Returns whether this option is a field path (ie, a nested field)"""
|
69
|
-
return self.field_path is not None
|
70
|
-
|
71
|
-
def inject_into_request(
|
72
|
-
self,
|
73
|
-
target: MutableMapping[str, Any],
|
74
|
-
value: Any,
|
75
|
-
config: Config,
|
76
|
-
) -> None:
|
77
|
-
"""
|
78
|
-
Inject a request option value into a target request structure using either field_name or field_path.
|
79
|
-
For non-body-json injection, only top-level field names are supported.
|
80
|
-
For body-json injection, both field names and nested field paths are supported.
|
81
|
-
|
82
|
-
Args:
|
83
|
-
target: The request structure to inject the value into
|
84
|
-
value: The value to inject
|
85
|
-
config: The config object to use for interpolation
|
86
|
-
"""
|
87
|
-
if self._is_field_path:
|
88
|
-
if self.inject_into != RequestOptionType.body_json:
|
89
|
-
raise ValueError(
|
90
|
-
"Nested field injection is only supported for body JSON injection. Please use a top-level field_name for other injection types."
|
91
|
-
)
|
92
|
-
|
93
|
-
assert self.field_path is not None # for type checker
|
94
|
-
current = target
|
95
|
-
# Convert path segments into strings, evaluating any interpolated segments
|
96
|
-
# Example: ["data", "{{ config[user_type] }}", "id"] -> ["data", "admin", "id"]
|
97
|
-
*path_parts, final_key = [
|
98
|
-
str(
|
99
|
-
segment.eval(config=config)
|
100
|
-
if isinstance(segment, InterpolatedString)
|
101
|
-
else segment
|
102
|
-
)
|
103
|
-
for segment in self.field_path
|
104
|
-
]
|
105
|
-
|
106
|
-
# Build a nested dictionary structure and set the final value at the deepest level
|
107
|
-
for part in path_parts:
|
108
|
-
current = current.setdefault(part, {})
|
109
|
-
current[final_key] = value
|
110
|
-
else:
|
111
|
-
# For non-nested fields, evaluate the field name if it's an interpolated string
|
112
|
-
key = (
|
113
|
-
self.field_name.eval(config=config)
|
114
|
-
if isinstance(self.field_name, InterpolatedString)
|
115
|
-
else self.field_name
|
116
|
-
)
|
117
|
-
target[str(key)] = value
|
38
|
+
self.field_name = InterpolatedString.create(self.field_name, parameters=parameters)
|
@@ -80,13 +80,12 @@ class DatetimeBasedRequestOptionsProvider(RequestOptionsProvider):
|
|
80
80
|
options: MutableMapping[str, Any] = {}
|
81
81
|
if not stream_slice:
|
82
82
|
return options
|
83
|
-
|
84
83
|
if self.start_time_option and self.start_time_option.inject_into == option_type:
|
85
|
-
|
86
|
-
|
87
|
-
|
84
|
+
options[self.start_time_option.field_name.eval(config=self.config)] = stream_slice.get( # type: ignore # field_name is always casted to an interpolated string
|
85
|
+
self._partition_field_start.eval(self.config)
|
86
|
+
)
|
88
87
|
if self.end_time_option and self.end_time_option.inject_into == option_type:
|
89
|
-
|
90
|
-
|
91
|
-
|
88
|
+
options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get( # type: ignore [union-attr]
|
89
|
+
self._partition_field_end.eval(self.config)
|
90
|
+
)
|
92
91
|
return options
|
@@ -6,7 +6,7 @@ from typing import Any, Iterable, Mapping, Optional
|
|
6
6
|
|
7
7
|
from typing_extensions import deprecated
|
8
8
|
|
9
|
-
from airbyte_cdk.
|
9
|
+
from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
|
10
10
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncPartition
|
11
11
|
from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
|
12
12
|
from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
|
@@ -16,7 +16,6 @@ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
|
|
16
16
|
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
17
17
|
from airbyte_cdk.sources.streams.core import StreamData
|
18
18
|
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
19
|
-
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
20
19
|
|
21
20
|
|
22
21
|
@deprecated(
|
@@ -57,9 +56,9 @@ class AsyncRetriever(Retriever):
|
|
57
56
|
|
58
57
|
return self.state
|
59
58
|
|
60
|
-
def
|
59
|
+
def _validate_and_get_stream_slice_jobs(
|
61
60
|
self, stream_slice: Optional[StreamSlice] = None
|
62
|
-
) ->
|
61
|
+
) -> Iterable[AsyncJob]:
|
63
62
|
"""
|
64
63
|
Validates the stream_slice argument and returns the partition from it.
|
65
64
|
|
@@ -73,12 +72,7 @@ class AsyncRetriever(Retriever):
|
|
73
72
|
AirbyteTracedException: If the stream_slice is not an instance of StreamSlice or if the partition is not present in the stream_slice.
|
74
73
|
|
75
74
|
"""
|
76
|
-
|
77
|
-
raise AirbyteTracedException(
|
78
|
-
message="Invalid arguments to AsyncJobRetriever.read_records: stream_slice is no optional. Please contact Airbyte Support",
|
79
|
-
failure_type=FailureType.system_error,
|
80
|
-
)
|
81
|
-
return stream_slice["partition"] # type: ignore # stream_slice["partition"] has been added as an AsyncPartition as part of stream_slices
|
75
|
+
return stream_slice.extra_fields.get("jobs", []) if stream_slice else []
|
82
76
|
|
83
77
|
def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
|
84
78
|
return self.stream_slicer.stream_slices()
|
@@ -89,8 +83,8 @@ class AsyncRetriever(Retriever):
|
|
89
83
|
stream_slice: Optional[StreamSlice] = None,
|
90
84
|
) -> Iterable[StreamData]:
|
91
85
|
stream_state: StreamState = self._get_stream_state()
|
92
|
-
|
93
|
-
records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(
|
86
|
+
jobs: Iterable[AsyncJob] = self._validate_and_get_stream_slice_jobs(stream_slice)
|
87
|
+
records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(jobs)
|
94
88
|
|
95
89
|
yield from self.record_selector.filter_and_transform(
|
96
90
|
all_data=records,
|
@@ -128,9 +128,6 @@ class SimpleRetriever(Retriever):
|
|
128
128
|
Returned merged mapping otherwise
|
129
129
|
"""
|
130
130
|
# FIXME we should eventually remove the usage of stream_state as part of the interpolation
|
131
|
-
|
132
|
-
is_body_json = paginator_method.__name__ == "get_request_body_json"
|
133
|
-
|
134
131
|
mappings = [
|
135
132
|
paginator_method(
|
136
133
|
stream_state=stream_state,
|
@@ -146,7 +143,7 @@ class SimpleRetriever(Retriever):
|
|
146
143
|
next_page_token=next_page_token,
|
147
144
|
)
|
148
145
|
)
|
149
|
-
return combine_mappings(mappings
|
146
|
+
return combine_mappings(mappings)
|
150
147
|
|
151
148
|
def _request_headers(
|
152
149
|
self,
|
@@ -163,7 +160,7 @@ class SimpleRetriever(Retriever):
|
|
163
160
|
stream_slice,
|
164
161
|
next_page_token,
|
165
162
|
self._paginator.get_request_headers,
|
166
|
-
self.
|
163
|
+
self.request_option_provider.get_request_headers,
|
167
164
|
)
|
168
165
|
if isinstance(headers, str):
|
169
166
|
raise ValueError("Request headers cannot be a string")
|
@@ -4,6 +4,7 @@
|
|
4
4
|
|
5
5
|
from airbyte_cdk.sources.declarative.schema.default_schema_loader import DefaultSchemaLoader
|
6
6
|
from airbyte_cdk.sources.declarative.schema.dynamic_schema_loader import (
|
7
|
+
ComplexFieldType,
|
7
8
|
DynamicSchemaLoader,
|
8
9
|
SchemaTypeIdentifier,
|
9
10
|
TypesMap,
|
@@ -18,6 +19,7 @@ __all__ = [
|
|
18
19
|
"SchemaLoader",
|
19
20
|
"InlineSchemaLoader",
|
20
21
|
"DynamicSchemaLoader",
|
22
|
+
"ComplexFieldType",
|
21
23
|
"TypesMap",
|
22
24
|
"SchemaTypeIdentifier",
|
23
25
|
]
|
@@ -18,7 +18,7 @@ from airbyte_cdk.sources.declarative.transformations import RecordTransformation
|
|
18
18
|
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
19
19
|
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
20
20
|
|
21
|
-
AIRBYTE_DATA_TYPES: Mapping[str,
|
21
|
+
AIRBYTE_DATA_TYPES: Mapping[str, MutableMapping[str, Any]] = {
|
22
22
|
"string": {"type": ["null", "string"]},
|
23
23
|
"boolean": {"type": ["null", "boolean"]},
|
24
24
|
"date": {"type": ["null", "string"], "format": "date"},
|
@@ -45,6 +45,25 @@ AIRBYTE_DATA_TYPES: Mapping[str, Mapping[str, Any]] = {
|
|
45
45
|
}
|
46
46
|
|
47
47
|
|
48
|
+
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
49
|
+
@dataclass(frozen=True)
|
50
|
+
class ComplexFieldType:
|
51
|
+
"""
|
52
|
+
Identifies complex field type
|
53
|
+
"""
|
54
|
+
|
55
|
+
field_type: str
|
56
|
+
items: Optional[Union[str, "ComplexFieldType"]] = None
|
57
|
+
|
58
|
+
def __post_init__(self) -> None:
|
59
|
+
"""
|
60
|
+
Enforces that `items` is only used when `field_type` is a array
|
61
|
+
"""
|
62
|
+
# `items_type` is valid only for array target types
|
63
|
+
if self.items and self.field_type != "array":
|
64
|
+
raise ValueError("'items' can only be used when 'field_type' is an array.")
|
65
|
+
|
66
|
+
|
48
67
|
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
49
68
|
@dataclass(frozen=True)
|
50
69
|
class TypesMap:
|
@@ -52,7 +71,7 @@ class TypesMap:
|
|
52
71
|
Represents a mapping between a current type and its corresponding target type.
|
53
72
|
"""
|
54
73
|
|
55
|
-
target_type: Union[List[str], str]
|
74
|
+
target_type: Union[List[str], str, ComplexFieldType]
|
56
75
|
current_type: Union[List[str], str]
|
57
76
|
condition: Optional[str]
|
58
77
|
|
@@ -135,8 +154,9 @@ class DynamicSchemaLoader(SchemaLoader):
|
|
135
154
|
transformed_properties = self._transform(properties, {})
|
136
155
|
|
137
156
|
return {
|
138
|
-
"$schema": "
|
157
|
+
"$schema": "https://json-schema.org/draft-07/schema#",
|
139
158
|
"type": "object",
|
159
|
+
"additionalProperties": True,
|
140
160
|
"properties": transformed_properties,
|
141
161
|
}
|
142
162
|
|
@@ -188,18 +208,37 @@ class DynamicSchemaLoader(SchemaLoader):
|
|
188
208
|
first_type = self._get_airbyte_type(mapped_field_type[0])
|
189
209
|
second_type = self._get_airbyte_type(mapped_field_type[1])
|
190
210
|
return {"oneOf": [first_type, second_type]}
|
211
|
+
|
191
212
|
elif isinstance(mapped_field_type, str):
|
192
213
|
return self._get_airbyte_type(mapped_field_type)
|
214
|
+
|
215
|
+
elif isinstance(mapped_field_type, ComplexFieldType):
|
216
|
+
return self._resolve_complex_type(mapped_field_type)
|
217
|
+
|
193
218
|
else:
|
194
219
|
raise ValueError(
|
195
220
|
f"Invalid data type. Available string or two items list of string. Got {mapped_field_type}."
|
196
221
|
)
|
197
222
|
|
223
|
+
def _resolve_complex_type(self, complex_type: ComplexFieldType) -> Mapping[str, Any]:
|
224
|
+
if not complex_type.items:
|
225
|
+
return self._get_airbyte_type(complex_type.field_type)
|
226
|
+
|
227
|
+
field_type = self._get_airbyte_type(complex_type.field_type)
|
228
|
+
|
229
|
+
field_type["items"] = (
|
230
|
+
self._get_airbyte_type(complex_type.items)
|
231
|
+
if isinstance(complex_type.items, str)
|
232
|
+
else self._resolve_complex_type(complex_type.items)
|
233
|
+
)
|
234
|
+
|
235
|
+
return field_type
|
236
|
+
|
198
237
|
def _replace_type_if_not_valid(
|
199
238
|
self,
|
200
239
|
field_type: Union[List[str], str],
|
201
240
|
raw_schema: MutableMapping[str, Any],
|
202
|
-
) -> Union[List[str], str]:
|
241
|
+
) -> Union[List[str], str, ComplexFieldType]:
|
203
242
|
"""
|
204
243
|
Replaces a field type if it matches a type mapping in `types_map`.
|
205
244
|
"""
|
@@ -216,7 +255,7 @@ class DynamicSchemaLoader(SchemaLoader):
|
|
216
255
|
return field_type
|
217
256
|
|
218
257
|
@staticmethod
|
219
|
-
def _get_airbyte_type(field_type: str) ->
|
258
|
+
def _get_airbyte_type(field_type: str) -> MutableMapping[str, Any]:
|
220
259
|
"""
|
221
260
|
Maps a field type to its corresponding Airbyte type definition.
|
222
261
|
"""
|
@@ -45,7 +45,7 @@ def format_http_message(
|
|
45
45
|
log_message["http"]["is_auxiliary"] = is_auxiliary # type: ignore [index]
|
46
46
|
if stream_name:
|
47
47
|
log_message["airbyte_cdk"] = {"stream": {"name": stream_name}}
|
48
|
-
return log_message # type: ignore
|
48
|
+
return log_message # type: ignore[return-value] # got "dict[str, object]", expected "dict[str, JsonType]"
|
49
49
|
|
50
50
|
|
51
51
|
def _normalize_body_string(body_str: Optional[Union[str, bytes]]) -> Optional[str]:
|