airbyte-cdk 6.9.1.dev0__py3-none-any.whl → 6.9.1.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +31 -25
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +2 -100
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +2 -53
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2 -95
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +0 -6
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +21 -95
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +1 -2
- airbyte_cdk/test/utils/manifest_only_fixtures.py +79 -0
- airbyte_cdk-6.9.1.dev2.dist-info/METADATA +306 -0
- {airbyte_cdk-6.9.1.dev0.dist-info → airbyte_cdk-6.9.1.dev2.dist-info}/RECORD +13 -15
- airbyte_cdk/sources/declarative/resolvers/__init__.py +0 -13
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +0 -55
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +0 -106
- airbyte_cdk-6.9.1.dev0.dist-info/METADATA +0 -108
- {airbyte_cdk-6.9.1.dev0.dist-info → airbyte_cdk-6.9.1.dev2.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.9.1.dev0.dist-info → airbyte_cdk-6.9.1.dev2.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.9.1.dev0.dist-info → airbyte_cdk-6.9.1.dev2.dist-info}/entry_points.txt +0 -0
@@ -86,10 +86,23 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
86
86
|
component_factory=component_factory,
|
87
87
|
)
|
88
88
|
|
89
|
-
# todo: We could remove state from initialization. Now that streams are grouped during the read(), a source
|
90
|
-
# no longer needs to store the original incoming state. But maybe there's an edge case?
|
91
89
|
self._state = state
|
92
90
|
|
91
|
+
self._concurrent_streams: Optional[List[AbstractStream]]
|
92
|
+
self._synchronous_streams: Optional[List[Stream]]
|
93
|
+
|
94
|
+
# If the connector command was SPEC, there is no incoming config, and we cannot instantiate streams because
|
95
|
+
# they might depend on it. Ideally we want to have a static method on this class to get the spec without
|
96
|
+
# any other arguments, but the existing entrypoint.py isn't designed to support this. Just noting this
|
97
|
+
# for our future improvements to the CDK.
|
98
|
+
if config:
|
99
|
+
self._concurrent_streams, self._synchronous_streams = self._group_streams(
|
100
|
+
config=config or {}
|
101
|
+
)
|
102
|
+
else:
|
103
|
+
self._concurrent_streams = None
|
104
|
+
self._synchronous_streams = None
|
105
|
+
|
93
106
|
concurrency_level_from_manifest = self._source_config.get("concurrency_level")
|
94
107
|
if concurrency_level_from_manifest:
|
95
108
|
concurrency_level_component = self._constructor.create_component(
|
@@ -123,20 +136,17 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
123
136
|
logger: logging.Logger,
|
124
137
|
config: Mapping[str, Any],
|
125
138
|
catalog: ConfiguredAirbyteCatalog,
|
126
|
-
state: Optional[List[AirbyteStateMessage]] = None,
|
139
|
+
state: Optional[Union[List[AirbyteStateMessage]]] = None,
|
127
140
|
) -> Iterator[AirbyteMessage]:
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
# the concurrent streams must be saved so that they can be removed from the catalog before starting
|
132
|
-
# synchronous streams
|
133
|
-
if len(concurrent_streams) > 0:
|
141
|
+
# ConcurrentReadProcessor pops streams that are finished being read so before syncing, the names of the concurrent
|
142
|
+
# streams must be saved so that they can be removed from the catalog before starting synchronous streams
|
143
|
+
if self._concurrent_streams:
|
134
144
|
concurrent_stream_names = set(
|
135
|
-
[concurrent_stream.name for concurrent_stream in
|
145
|
+
[concurrent_stream.name for concurrent_stream in self._concurrent_streams]
|
136
146
|
)
|
137
147
|
|
138
148
|
selected_concurrent_streams = self._select_streams(
|
139
|
-
streams=
|
149
|
+
streams=self._concurrent_streams, configured_catalog=catalog
|
140
150
|
)
|
141
151
|
# It would appear that passing in an empty set of streams causes an infinite loop in ConcurrentReadProcessor.
|
142
152
|
# This is also evident in concurrent_source_adapter.py so I'll leave this out of scope to fix for now
|
@@ -155,7 +165,8 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
155
165
|
yield from super().read(logger, config, filtered_catalog, state)
|
156
166
|
|
157
167
|
def discover(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCatalog:
|
158
|
-
concurrent_streams
|
168
|
+
concurrent_streams = self._concurrent_streams or []
|
169
|
+
synchronous_streams = self._synchronous_streams or []
|
159
170
|
return AirbyteCatalog(
|
160
171
|
streams=[
|
161
172
|
stream.as_airbyte_stream() for stream in concurrent_streams + synchronous_streams
|
@@ -181,13 +192,9 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
181
192
|
|
182
193
|
state_manager = ConnectorStateManager(state=self._state) # type: ignore # state is always in the form of List[AirbyteStateMessage]. The ConnectorStateManager should use generics, but this can be done later
|
183
194
|
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
self._source_config, config
|
188
|
-
)
|
189
|
-
|
190
|
-
name_to_stream_mapping = {stream["name"]: stream for stream in streams}
|
195
|
+
name_to_stream_mapping = {
|
196
|
+
stream["name"]: stream for stream in self.resolved_manifest["streams"]
|
197
|
+
}
|
191
198
|
|
192
199
|
for declarative_stream in self.streams(config=config):
|
193
200
|
# Some low-code sources use a combination of DeclarativeStream and regular Python streams. We can't inspect
|
@@ -195,7 +202,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
195
202
|
# so we need to treat them as synchronous
|
196
203
|
if (
|
197
204
|
isinstance(declarative_stream, DeclarativeStream)
|
198
|
-
and name_to_stream_mapping[declarative_stream.name]
|
205
|
+
and name_to_stream_mapping[declarative_stream.name].get("retriever")["type"]
|
199
206
|
== "SimpleRetriever"
|
200
207
|
):
|
201
208
|
incremental_sync_component_definition = name_to_stream_mapping[
|
@@ -204,7 +211,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
204
211
|
|
205
212
|
partition_router_component_definition = (
|
206
213
|
name_to_stream_mapping[declarative_stream.name]
|
207
|
-
.get("retriever"
|
214
|
+
.get("retriever")
|
208
215
|
.get("partition_router")
|
209
216
|
)
|
210
217
|
is_without_partition_router_or_cursor = not bool(
|
@@ -226,7 +233,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
226
233
|
cursor = self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
|
227
234
|
state_manager=state_manager,
|
228
235
|
model_type=DatetimeBasedCursorModel,
|
229
|
-
component_definition=incremental_sync_component_definition,
|
236
|
+
component_definition=incremental_sync_component_definition,
|
230
237
|
stream_name=declarative_stream.name,
|
231
238
|
stream_namespace=declarative_stream.namespace,
|
232
239
|
config=config or {},
|
@@ -309,11 +316,10 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
309
316
|
def _is_datetime_incremental_without_partition_routing(
|
310
317
|
self,
|
311
318
|
declarative_stream: DeclarativeStream,
|
312
|
-
incremental_sync_component_definition: Mapping[str, Any]
|
319
|
+
incremental_sync_component_definition: Mapping[str, Any],
|
313
320
|
) -> bool:
|
314
321
|
return (
|
315
|
-
incremental_sync_component_definition
|
316
|
-
and bool(incremental_sync_component_definition)
|
322
|
+
bool(incremental_sync_component_definition)
|
317
323
|
and incremental_sync_component_definition.get("type", "")
|
318
324
|
== DatetimeBasedCursorModel.__name__
|
319
325
|
and self._stream_supports_concurrent_partition_processing(
|
@@ -7,12 +7,8 @@ version: 1.0.0
|
|
7
7
|
required:
|
8
8
|
- type
|
9
9
|
- check
|
10
|
+
- streams
|
10
11
|
- version
|
11
|
-
anyOf:
|
12
|
-
- required:
|
13
|
-
- streams
|
14
|
-
- required:
|
15
|
-
- dynamic_streams
|
16
12
|
properties:
|
17
13
|
type:
|
18
14
|
type: string
|
@@ -23,10 +19,6 @@ properties:
|
|
23
19
|
type: array
|
24
20
|
items:
|
25
21
|
"$ref": "#/definitions/DeclarativeStream"
|
26
|
-
dynamic_streams:
|
27
|
-
type: array
|
28
|
-
items:
|
29
|
-
"$ref": "#/definitions/DynamicDeclarativeStream"
|
30
22
|
version:
|
31
23
|
type: string
|
32
24
|
description: The version of the Airbyte CDK used to build and test the source.
|
@@ -1329,7 +1321,7 @@ definitions:
|
|
1329
1321
|
type: array
|
1330
1322
|
items:
|
1331
1323
|
- type: string
|
1332
|
-
|
1324
|
+
interpolation_content:
|
1333
1325
|
- config
|
1334
1326
|
examples:
|
1335
1327
|
- ["data"]
|
@@ -2903,96 +2895,6 @@ definitions:
|
|
2903
2895
|
$parameters:
|
2904
2896
|
type: object
|
2905
2897
|
additionalProperties: true
|
2906
|
-
ComponentMappingDefinition:
|
2907
|
-
title: Component Mapping Definition
|
2908
|
-
description: (This component is experimental. Use at your own risk.) Specifies a mapping definition to update or add fields in a record or configuration. This allows dynamic mapping of data by interpolating values into the template based on provided contexts.
|
2909
|
-
type: object
|
2910
|
-
required:
|
2911
|
-
- type
|
2912
|
-
- field_path
|
2913
|
-
- value
|
2914
|
-
properties:
|
2915
|
-
type:
|
2916
|
-
type: string
|
2917
|
-
enum: [ComponentMappingDefinition]
|
2918
|
-
field_path:
|
2919
|
-
title: Field Path
|
2920
|
-
description: A list of potentially nested fields indicating the full path where value will be added or updated.
|
2921
|
-
type: array
|
2922
|
-
items:
|
2923
|
-
- type: string
|
2924
|
-
interpolation_context:
|
2925
|
-
- config
|
2926
|
-
- components_values
|
2927
|
-
- stream_template_config
|
2928
|
-
examples:
|
2929
|
-
- ["data"]
|
2930
|
-
- ["data", "records"]
|
2931
|
-
- ["data", "{{ parameters.name }}"]
|
2932
|
-
- ["data", "*", "record"]
|
2933
|
-
value:
|
2934
|
-
title: Value
|
2935
|
-
description: The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.
|
2936
|
-
type: string
|
2937
|
-
interpolation_context:
|
2938
|
-
- config
|
2939
|
-
- stream_template_config
|
2940
|
-
- components_values
|
2941
|
-
examples:
|
2942
|
-
- "{{ components_values['updates'] }}"
|
2943
|
-
- "{{ components_values['MetaData']['LastUpdatedTime'] }}"
|
2944
|
-
- "{{ config['segment_id'] }}"
|
2945
|
-
value_type:
|
2946
|
-
title: Value Type
|
2947
|
-
description: The expected data type of the value. If omitted, the type will be inferred from the value provided.
|
2948
|
-
"$ref": "#/definitions/ValueType"
|
2949
|
-
$parameters:
|
2950
|
-
type: object
|
2951
|
-
additionalProperties: true
|
2952
|
-
HttpComponentsResolver:
|
2953
|
-
type: object
|
2954
|
-
description: (This component is experimental. Use at your own risk.) Component resolve and populates stream templates with components fetched via an HTTP retriever.
|
2955
|
-
properties:
|
2956
|
-
type:
|
2957
|
-
type: string
|
2958
|
-
enum: [HttpComponentsResolver]
|
2959
|
-
retriever:
|
2960
|
-
title: Retriever
|
2961
|
-
description: Component used to coordinate how records are extracted across stream slices and request pages.
|
2962
|
-
anyOf:
|
2963
|
-
- "$ref": "#/definitions/AsyncRetriever"
|
2964
|
-
- "$ref": "#/definitions/CustomRetriever"
|
2965
|
-
- "$ref": "#/definitions/SimpleRetriever"
|
2966
|
-
components_mapping:
|
2967
|
-
type: array
|
2968
|
-
items:
|
2969
|
-
"$ref": "#/definitions/ComponentMappingDefinition"
|
2970
|
-
$parameters:
|
2971
|
-
type: object
|
2972
|
-
additionalProperties: true
|
2973
|
-
required:
|
2974
|
-
- type
|
2975
|
-
- retriever
|
2976
|
-
- components_mapping
|
2977
|
-
DynamicDeclarativeStream:
|
2978
|
-
type: object
|
2979
|
-
description: (This component is experimental. Use at your own risk.) A component that described how will be created declarative streams based on stream template.
|
2980
|
-
properties:
|
2981
|
-
type:
|
2982
|
-
type: string
|
2983
|
-
enum: [DynamicDeclarativeStream]
|
2984
|
-
stream_template:
|
2985
|
-
title: Stream Template
|
2986
|
-
description: Reference to the stream template.
|
2987
|
-
"$ref": "#/definitions/DeclarativeStream"
|
2988
|
-
components_resolver:
|
2989
|
-
title: Components Resolver
|
2990
|
-
description: Component resolve and populates stream templates with components values.
|
2991
|
-
"$ref": "#/definitions/HttpComponentsResolver"
|
2992
|
-
required:
|
2993
|
-
- type
|
2994
|
-
- stream_template
|
2995
|
-
- components_resolver
|
2996
2898
|
interpolation:
|
2997
2899
|
variables:
|
2998
2900
|
- title: config
|
@@ -39,7 +39,6 @@ from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import
|
|
39
39
|
from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
|
40
40
|
ModelToComponentFactory,
|
41
41
|
)
|
42
|
-
from airbyte_cdk.sources.declarative.resolvers import COMPONENTS_RESOLVER_TYPE_MAPPING
|
43
42
|
from airbyte_cdk.sources.message import MessageRepository
|
44
43
|
from airbyte_cdk.sources.streams.core import Stream
|
45
44
|
from airbyte_cdk.sources.types import ConnectionDefinition
|
@@ -121,10 +120,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
121
120
|
self._emit_manifest_debug_message(
|
122
121
|
extra_args={"source_name": self.name, "parsed_config": json.dumps(self._source_config)}
|
123
122
|
)
|
124
|
-
|
125
|
-
stream_configs = self._stream_configs(self._source_config) + self._dynamic_stream_configs(
|
126
|
-
self._source_config, config
|
127
|
-
)
|
123
|
+
stream_configs = self._stream_configs(self._source_config)
|
128
124
|
|
129
125
|
source_streams = [
|
130
126
|
self._constructor.create_component(
|
@@ -238,8 +234,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
238
234
|
)
|
239
235
|
|
240
236
|
streams = self._source_config.get("streams")
|
241
|
-
|
242
|
-
if not (streams or dynamic_streams):
|
237
|
+
if not streams:
|
243
238
|
raise ValidationError(
|
244
239
|
f"A valid manifest should have at least one stream defined. Got {streams}"
|
245
240
|
)
|
@@ -308,51 +303,5 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
308
303
|
s["type"] = "DeclarativeStream"
|
309
304
|
return stream_configs
|
310
305
|
|
311
|
-
def _dynamic_stream_configs(
|
312
|
-
self, manifest: Mapping[str, Any], config: Mapping[str, Any]
|
313
|
-
) -> List[Dict[str, Any]]:
|
314
|
-
dynamic_stream_definitions: List[Dict[str, Any]] = manifest.get("dynamic_streams", [])
|
315
|
-
dynamic_stream_configs: List[Dict[str, Any]] = []
|
316
|
-
|
317
|
-
for dynamic_definition in dynamic_stream_definitions:
|
318
|
-
components_resolver_config = dynamic_definition["components_resolver"]
|
319
|
-
|
320
|
-
if not components_resolver_config:
|
321
|
-
raise ValueError(
|
322
|
-
f"Missing 'components_resolver' in dynamic definition: {dynamic_definition}"
|
323
|
-
)
|
324
|
-
|
325
|
-
resolver_type = components_resolver_config.get("type")
|
326
|
-
if not resolver_type:
|
327
|
-
raise ValueError(
|
328
|
-
f"Missing 'type' in components resolver configuration: {components_resolver_config}"
|
329
|
-
)
|
330
|
-
|
331
|
-
if resolver_type not in COMPONENTS_RESOLVER_TYPE_MAPPING:
|
332
|
-
raise ValueError(
|
333
|
-
f"Invalid components resolver type '{resolver_type}'. "
|
334
|
-
f"Expected one of {list(COMPONENTS_RESOLVER_TYPE_MAPPING.keys())}."
|
335
|
-
)
|
336
|
-
|
337
|
-
if "retriever" in components_resolver_config:
|
338
|
-
components_resolver_config["retriever"]["requester"]["use_cache"] = True
|
339
|
-
|
340
|
-
# Create a resolver for dynamic components based on type
|
341
|
-
components_resolver = self._constructor.create_component(
|
342
|
-
COMPONENTS_RESOLVER_TYPE_MAPPING[resolver_type], components_resolver_config, config
|
343
|
-
)
|
344
|
-
|
345
|
-
stream_template_config = dynamic_definition["stream_template"]
|
346
|
-
|
347
|
-
for dynamic_stream in components_resolver.resolve_components(
|
348
|
-
stream_template_config=stream_template_config
|
349
|
-
):
|
350
|
-
if "type" not in dynamic_stream:
|
351
|
-
dynamic_stream["type"] = "DeclarativeStream"
|
352
|
-
|
353
|
-
dynamic_stream_configs.append(dynamic_stream)
|
354
|
-
|
355
|
-
return dynamic_stream_configs
|
356
|
-
|
357
306
|
def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None:
|
358
307
|
self.logger.debug("declarative source created from manifest", extra=extra_args)
|
@@ -1158,37 +1158,6 @@ class WaitUntilTimeFromHeader(BaseModel):
|
|
1158
1158
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1159
1159
|
|
1160
1160
|
|
1161
|
-
class ComponentMappingDefinition(BaseModel):
|
1162
|
-
type: Literal["ComponentMappingDefinition"]
|
1163
|
-
field_path: List[str] = Field(
|
1164
|
-
...,
|
1165
|
-
description="A list of potentially nested fields indicating the full path where value will be added or updated.",
|
1166
|
-
examples=[
|
1167
|
-
["data"],
|
1168
|
-
["data", "records"],
|
1169
|
-
["data", "{{ parameters.name }}"],
|
1170
|
-
["data", "*", "record"],
|
1171
|
-
],
|
1172
|
-
title="Field Path",
|
1173
|
-
)
|
1174
|
-
value: str = Field(
|
1175
|
-
...,
|
1176
|
-
description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.",
|
1177
|
-
examples=[
|
1178
|
-
"{{ components_values['updates'] }}",
|
1179
|
-
"{{ components_values['MetaData']['LastUpdatedTime'] }}",
|
1180
|
-
"{{ config['segment_id'] }}",
|
1181
|
-
],
|
1182
|
-
title="Value",
|
1183
|
-
)
|
1184
|
-
value_type: Optional[ValueType] = Field(
|
1185
|
-
None,
|
1186
|
-
description="The expected data type of the value. If omitted, the type will be inferred from the value provided.",
|
1187
|
-
title="Value Type",
|
1188
|
-
)
|
1189
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1190
|
-
|
1191
|
-
|
1192
1161
|
class AddedFieldDefinition(BaseModel):
|
1193
1162
|
type: Literal["AddedFieldDefinition"]
|
1194
1163
|
path: List[str] = Field(
|
@@ -1486,40 +1455,13 @@ class CompositeErrorHandler(BaseModel):
|
|
1486
1455
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1487
1456
|
|
1488
1457
|
|
1489
|
-
class
|
1458
|
+
class DeclarativeSource(BaseModel):
|
1490
1459
|
class Config:
|
1491
1460
|
extra = Extra.forbid
|
1492
1461
|
|
1493
1462
|
type: Literal["DeclarativeSource"]
|
1494
1463
|
check: CheckStream
|
1495
1464
|
streams: List[DeclarativeStream]
|
1496
|
-
dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None
|
1497
|
-
version: str = Field(
|
1498
|
-
...,
|
1499
|
-
description="The version of the Airbyte CDK used to build and test the source.",
|
1500
|
-
)
|
1501
|
-
schemas: Optional[Schemas] = None
|
1502
|
-
definitions: Optional[Dict[str, Any]] = None
|
1503
|
-
spec: Optional[Spec] = None
|
1504
|
-
concurrency_level: Optional[ConcurrencyLevel] = None
|
1505
|
-
metadata: Optional[Dict[str, Any]] = Field(
|
1506
|
-
None,
|
1507
|
-
description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
|
1508
|
-
)
|
1509
|
-
description: Optional[str] = Field(
|
1510
|
-
None,
|
1511
|
-
description="A description of the connector. It will be presented on the Source documentation page.",
|
1512
|
-
)
|
1513
|
-
|
1514
|
-
|
1515
|
-
class DeclarativeSource2(BaseModel):
|
1516
|
-
class Config:
|
1517
|
-
extra = Extra.forbid
|
1518
|
-
|
1519
|
-
type: Literal["DeclarativeSource"]
|
1520
|
-
check: CheckStream
|
1521
|
-
streams: Optional[List[DeclarativeStream]] = None
|
1522
|
-
dynamic_streams: List[DynamicDeclarativeStream]
|
1523
1465
|
version: str = Field(
|
1524
1466
|
...,
|
1525
1467
|
description="The version of the Airbyte CDK used to build and test the source.",
|
@@ -1538,17 +1480,6 @@ class DeclarativeSource2(BaseModel):
|
|
1538
1480
|
)
|
1539
1481
|
|
1540
1482
|
|
1541
|
-
class DeclarativeSource(BaseModel):
|
1542
|
-
class Config:
|
1543
|
-
extra = Extra.forbid
|
1544
|
-
|
1545
|
-
__root__: Union[DeclarativeSource1, DeclarativeSource2] = Field(
|
1546
|
-
...,
|
1547
|
-
description="An API source that extracts data according to its declarative components.",
|
1548
|
-
title="DeclarativeSource",
|
1549
|
-
)
|
1550
|
-
|
1551
|
-
|
1552
1483
|
class SelectiveAuthenticator(BaseModel):
|
1553
1484
|
class Config:
|
1554
1485
|
extra = Extra.allow
|
@@ -1952,32 +1883,8 @@ class SubstreamPartitionRouter(BaseModel):
|
|
1952
1883
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1953
1884
|
|
1954
1885
|
|
1955
|
-
class HttpComponentsResolver(BaseModel):
|
1956
|
-
type: Literal["HttpComponentsResolver"]
|
1957
|
-
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
|
1958
|
-
...,
|
1959
|
-
description="Component used to coordinate how records are extracted across stream slices and request pages.",
|
1960
|
-
title="Retriever",
|
1961
|
-
)
|
1962
|
-
components_mapping: List[ComponentMappingDefinition]
|
1963
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1964
|
-
|
1965
|
-
|
1966
|
-
class DynamicDeclarativeStream(BaseModel):
|
1967
|
-
type: Literal["DynamicDeclarativeStream"]
|
1968
|
-
stream_template: DeclarativeStream = Field(
|
1969
|
-
..., description="Reference to the stream template.", title="Stream Template"
|
1970
|
-
)
|
1971
|
-
components_resolver: HttpComponentsResolver = Field(
|
1972
|
-
...,
|
1973
|
-
description="Component resolve and populates stream templates with components values.",
|
1974
|
-
title="Components Resolver",
|
1975
|
-
)
|
1976
|
-
|
1977
|
-
|
1978
1886
|
CompositeErrorHandler.update_forward_refs()
|
1979
|
-
|
1980
|
-
DeclarativeSource2.update_forward_refs()
|
1887
|
+
DeclarativeSource.update_forward_refs()
|
1981
1888
|
SelectiveAuthenticator.update_forward_refs()
|
1982
1889
|
DeclarativeStream.update_forward_refs()
|
1983
1890
|
SessionTokenAuthenticator.update_forward_refs()
|
@@ -31,12 +31,6 @@ DEFAULT_MODEL_TYPES: Mapping[str, str] = {
|
|
31
31
|
# DeclarativeStream
|
32
32
|
"DeclarativeStream.retriever": "SimpleRetriever",
|
33
33
|
"DeclarativeStream.schema_loader": "JsonFileSchemaLoader",
|
34
|
-
# DynamicDeclarativeStream
|
35
|
-
"DynamicDeclarativeStream.stream_template": "DeclarativeStream",
|
36
|
-
"DynamicDeclarativeStream.components_resolver": "HttpComponentsResolver",
|
37
|
-
# HttpComponentsResolver
|
38
|
-
"HttpComponentsResolver.retriever": "SimpleRetriever",
|
39
|
-
"HttpComponentsResolver.components_mapping": "ComponentMappingDefinition",
|
40
34
|
# DefaultErrorHandler
|
41
35
|
"DefaultErrorHandler.response_filters": "HttpResponseFilter",
|
42
36
|
# DefaultPaginator
|
@@ -119,9 +119,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
119
119
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
120
120
|
CheckStream as CheckStreamModel,
|
121
121
|
)
|
122
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
123
|
-
ComponentMappingDefinition as ComponentMappingDefinitionModel,
|
124
|
-
)
|
125
122
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
126
123
|
CompositeErrorHandler as CompositeErrorHandlerModel,
|
127
124
|
)
|
@@ -194,9 +191,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
194
191
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
195
192
|
GzipJsonDecoder as GzipJsonDecoderModel,
|
196
193
|
)
|
197
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
198
|
-
HttpComponentsResolver as HttpComponentsResolverModel,
|
199
|
-
)
|
200
194
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
201
195
|
HttpRequester as HttpRequesterModel,
|
202
196
|
)
|
@@ -304,7 +298,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
304
298
|
from airbyte_cdk.sources.declarative.partition_routers import (
|
305
299
|
CartesianProductStreamSlicer,
|
306
300
|
ListPartitionRouter,
|
307
|
-
PartitionRouter,
|
308
301
|
SinglePartitionRouter,
|
309
302
|
SubstreamPartitionRouter,
|
310
303
|
)
|
@@ -345,10 +338,6 @@ from airbyte_cdk.sources.declarative.requesters.request_options import (
|
|
345
338
|
)
|
346
339
|
from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
|
347
340
|
from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
|
348
|
-
from airbyte_cdk.sources.declarative.resolvers import (
|
349
|
-
ComponentMappingDefinition,
|
350
|
-
HttpComponentsResolver,
|
351
|
-
)
|
352
341
|
from airbyte_cdk.sources.declarative.retrievers import (
|
353
342
|
AsyncRetriever,
|
354
343
|
SimpleRetriever,
|
@@ -478,8 +467,6 @@ class ModelToComponentFactory:
|
|
478
467
|
WaitTimeFromHeaderModel: self.create_wait_time_from_header,
|
479
468
|
WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header,
|
480
469
|
AsyncRetrieverModel: self.create_async_retriever,
|
481
|
-
HttpComponentsResolverModel: self.create_http_components_resolver,
|
482
|
-
ComponentMappingDefinitionModel: self.create_components_mapping_definition,
|
483
470
|
}
|
484
471
|
|
485
472
|
# Needed for the case where we need to perform a second parse on the fields of a custom component
|
@@ -1294,20 +1281,19 @@ class ModelToComponentFactory:
|
|
1294
1281
|
parameters=model.parameters or {},
|
1295
1282
|
)
|
1296
1283
|
|
1297
|
-
def
|
1298
|
-
self,
|
1299
|
-
|
1300
|
-
|
1301
|
-
) -> Optional[PartitionRouter]:
|
1284
|
+
def _merge_stream_slicers(
|
1285
|
+
self, model: DeclarativeStreamModel, config: Config
|
1286
|
+
) -> Optional[StreamSlicer]:
|
1287
|
+
stream_slicer = None
|
1302
1288
|
if (
|
1303
|
-
hasattr(model, "partition_router")
|
1304
|
-
and isinstance(model, SimpleRetrieverModel)
|
1305
|
-
and model.partition_router
|
1289
|
+
hasattr(model.retriever, "partition_router")
|
1290
|
+
and isinstance(model.retriever, SimpleRetrieverModel)
|
1291
|
+
and model.retriever.partition_router
|
1306
1292
|
):
|
1307
|
-
stream_slicer_model = model.partition_router
|
1293
|
+
stream_slicer_model = model.retriever.partition_router
|
1308
1294
|
|
1309
1295
|
if isinstance(stream_slicer_model, list):
|
1310
|
-
|
1296
|
+
stream_slicer = CartesianProductStreamSlicer(
|
1311
1297
|
[
|
1312
1298
|
self._create_component_from_model(model=slicer, config=config)
|
1313
1299
|
for slicer in stream_slicer_model
|
@@ -1315,24 +1301,9 @@ class ModelToComponentFactory:
|
|
1315
1301
|
parameters={},
|
1316
1302
|
)
|
1317
1303
|
else:
|
1318
|
-
|
1319
|
-
|
1320
|
-
|
1321
|
-
|
1322
|
-
def _build_resumable_cursor_from_paginator(
|
1323
|
-
self,
|
1324
|
-
model: Union[AsyncRetrieverModel, CustomRetrieverModel, SimpleRetrieverModel],
|
1325
|
-
stream_slicer: Optional[StreamSlicer],
|
1326
|
-
) -> Optional[StreamSlicer]:
|
1327
|
-
if hasattr(model, "paginator") and model.paginator and not stream_slicer:
|
1328
|
-
# For the regular Full-Refresh streams, we use the high level `ResumableFullRefreshCursor`
|
1329
|
-
return ResumableFullRefreshCursor(parameters={})
|
1330
|
-
return None
|
1331
|
-
|
1332
|
-
def _merge_stream_slicers(
|
1333
|
-
self, model: DeclarativeStreamModel, config: Config
|
1334
|
-
) -> Optional[StreamSlicer]:
|
1335
|
-
stream_slicer = self._build_stream_slicer_from_partition_router(model.retriever, config)
|
1304
|
+
stream_slicer = self._create_component_from_model(
|
1305
|
+
model=stream_slicer_model, config=config
|
1306
|
+
)
|
1336
1307
|
|
1337
1308
|
if model.incremental_sync and stream_slicer:
|
1338
1309
|
incremental_sync_model = model.incremental_sync
|
@@ -1375,7 +1346,15 @@ class ModelToComponentFactory:
|
|
1375
1346
|
),
|
1376
1347
|
partition_router=stream_slicer,
|
1377
1348
|
)
|
1378
|
-
|
1349
|
+
elif (
|
1350
|
+
hasattr(model.retriever, "paginator")
|
1351
|
+
and model.retriever.paginator
|
1352
|
+
and not stream_slicer
|
1353
|
+
):
|
1354
|
+
# For the regular Full-Refresh streams, we use the high level `ResumableFullRefreshCursor`
|
1355
|
+
return ResumableFullRefreshCursor(parameters={})
|
1356
|
+
else:
|
1357
|
+
return None
|
1379
1358
|
|
1380
1359
|
def create_default_error_handler(
|
1381
1360
|
self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any
|
@@ -2239,56 +2218,3 @@ class ModelToComponentFactory:
|
|
2239
2218
|
|
2240
2219
|
def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level:
|
2241
2220
|
return Level.DEBUG if emit_connector_builder_messages else Level.INFO
|
2242
|
-
|
2243
|
-
@staticmethod
|
2244
|
-
def create_components_mapping_definition(
|
2245
|
-
model: ComponentMappingDefinitionModel, config: Config, **kwargs: Any
|
2246
|
-
) -> ComponentMappingDefinition:
|
2247
|
-
interpolated_value = InterpolatedString.create(
|
2248
|
-
model.value, parameters=model.parameters or {}
|
2249
|
-
)
|
2250
|
-
field_path = [
|
2251
|
-
InterpolatedString.create(path, parameters=model.parameters or {})
|
2252
|
-
for path in model.field_path
|
2253
|
-
]
|
2254
|
-
return ComponentMappingDefinition(
|
2255
|
-
field_path=field_path, # type: ignore[arg-type] # field_path can be str and InterpolatedString
|
2256
|
-
value=interpolated_value,
|
2257
|
-
value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type),
|
2258
|
-
parameters=model.parameters or {},
|
2259
|
-
)
|
2260
|
-
|
2261
|
-
def create_http_components_resolver(
|
2262
|
-
self, model: HttpComponentsResolverModel, config: Config
|
2263
|
-
) -> Any:
|
2264
|
-
stream_slicer = self._build_stream_slicer_from_partition_router(model.retriever, config)
|
2265
|
-
combined_slicers = self._build_resumable_cursor_from_paginator(
|
2266
|
-
model.retriever, stream_slicer
|
2267
|
-
)
|
2268
|
-
|
2269
|
-
retriever = self._create_component_from_model(
|
2270
|
-
model=model.retriever,
|
2271
|
-
config=config,
|
2272
|
-
name="",
|
2273
|
-
primary_key=None,
|
2274
|
-
stream_slicer=combined_slicers,
|
2275
|
-
transformations=[],
|
2276
|
-
)
|
2277
|
-
|
2278
|
-
components_mapping = [
|
2279
|
-
self._create_component_from_model(
|
2280
|
-
model=components_mapping_definition_model,
|
2281
|
-
value_type=ModelToComponentFactory._json_schema_type_name_to_type(
|
2282
|
-
components_mapping_definition_model.value_type
|
2283
|
-
),
|
2284
|
-
config=config,
|
2285
|
-
)
|
2286
|
-
for components_mapping_definition_model in model.components_mapping
|
2287
|
-
]
|
2288
|
-
|
2289
|
-
return HttpComponentsResolver(
|
2290
|
-
retriever=retriever,
|
2291
|
-
config=config,
|
2292
|
-
components_mapping=components_mapping,
|
2293
|
-
parameters=model.parameters or {},
|
2294
|
-
)
|
@@ -6,6 +6,5 @@ from airbyte_cdk.sources.declarative.partition_routers.cartesian_product_stream_
|
|
6
6
|
from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import ListPartitionRouter
|
7
7
|
from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import SinglePartitionRouter
|
8
8
|
from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter
|
9
|
-
from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
|
10
9
|
|
11
|
-
__all__ = ["CartesianProductStreamSlicer", "ListPartitionRouter", "SinglePartitionRouter", "SubstreamPartitionRouter"
|
10
|
+
__all__ = ["CartesianProductStreamSlicer", "ListPartitionRouter", "SinglePartitionRouter", "SubstreamPartitionRouter"]
|