airbyte-cdk 6.8.1rc10__py3-none-any.whl → 6.8.2.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +87 -25
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +2 -100
- airbyte_cdk/sources/declarative/extractors/record_filter.py +3 -5
- airbyte_cdk/sources/declarative/incremental/__init__.py +3 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +270 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +9 -0
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +2 -53
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2 -95
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +0 -6
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +87 -106
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +1 -2
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +1 -1
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +0 -1
- airbyte_cdk/sources/streams/concurrent/cursor.py +9 -0
- {airbyte_cdk-6.8.1rc10.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.8.1rc10.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/RECORD +19 -21
- airbyte_cdk/sources/declarative/resolvers/__init__.py +0 -13
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +0 -55
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +0 -106
- {airbyte_cdk-6.8.1rc10.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.8.1rc10.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.8.1rc10.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/entry_points.txt +0 -0
@@ -62,11 +62,11 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQr
|
|
62
62
|
airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
|
63
63
|
airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
|
64
64
|
airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
|
65
|
-
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=
|
65
|
+
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=PWqtQ6xzRZiM0XrMO_zCJjl9tvbFMJMwSec0nN2ZekA,26434
|
66
66
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
|
67
67
|
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
|
68
68
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
69
|
-
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=
|
69
|
+
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=Q6zOEKvgAjMUsu89VmNteMhqdP0XwO9WWfA6-rMVn6A,120545
|
70
70
|
airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
|
71
71
|
airbyte_cdk/sources/declarative/declarative_stream.py,sha256=JRyNeOIpsFu4ztVZsN6sncqUEIqIE-bUkD2TPgbMgk0,10375
|
72
72
|
airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=hNlhaB5FjNC6IfJyglj5ZJWkYD2nEAukMDmzRz5PC6o,671
|
@@ -80,14 +80,15 @@ airbyte_cdk/sources/declarative/extractors/__init__.py,sha256=YFuL4D4RuuB8E1DNSb
|
|
80
80
|
airbyte_cdk/sources/declarative/extractors/dpath_extractor.py,sha256=wR4Ol4MG2lt5UlqXF5EU_k7qa5cN4_-luu3PJ1PlO3A,3131
|
81
81
|
airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8VwkjS0xD_u350Km3SiYP7hpOOgiLg5o,1169
|
82
82
|
airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
|
83
|
-
airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=
|
83
|
+
airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
|
84
84
|
airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=AkXPOWyp741cpYLBl9AbmVmOQmQ2BzZ2XjgsMEB6gGc,6583
|
85
85
|
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=LhqGDfX06_dDYLKsIVnwQ_nAWCln-v8PV7Wgt_QVeTI,6533
|
86
|
-
airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=
|
86
|
+
airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=zEERPIXz1WxCJypqlSXZCFIpT4-mIsjzRdmFlX2-nMg,1210
|
87
|
+
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=-ECXZbDh3nw7G4mBncsTT_68LWQvS8TySIgckBTZZQQ,11899
|
87
88
|
airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=_UzUnSIUsDbRgbFTXgSyZEFb4ws-KdhdQPWO8mFbV7U,22028
|
88
89
|
airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
|
89
90
|
airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=3_EEZop94bMitZaJd2PF5Q2Xt9v94tYg7p7YJz8tAFc,15869
|
90
|
-
airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py,sha256=
|
91
|
+
airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py,sha256=cdk4gSuYQmqcxxIOclhms6cnI1qm-FrSu7lmZULxOPM,16199
|
91
92
|
airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py,sha256=2YBOA2NnwAeIKlIhSwUB_W-FaGnPcmrG_liY7b4mV2Y,8365
|
92
93
|
airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py,sha256=10LFv1QPM-agVKl6eaANmEBOfd7gZgBrkoTcMggsieQ,4809
|
93
94
|
airbyte_cdk/sources/declarative/interpolation/__init__.py,sha256=tjUJkn3B-iZ-p7RP2c3dVZejrGiQeooGmS5ibWTuUL4,437
|
@@ -99,18 +100,18 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=LYEZ
|
|
99
100
|
airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=-V5UddGm69UKEB6o_O1EIES9kfY8FV_X4Ji8w1yOuSA,981
|
100
101
|
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=gcihTEnfD_6sUivxOomoY5r7VMAGqVVnK_HEsid9Y5k,6605
|
101
102
|
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=jf24RK-1fBhTYDpcGEakZtGNNJfG5NS8CCF5bEgNmRo,3977
|
102
|
-
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256
|
103
|
+
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=FEOmFo2mwdpmO8pH9jnw-sUAnijjuigZWYqH_0Gq9oQ,12919
|
103
104
|
airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
104
105
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
|
105
106
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
106
107
|
airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
|
107
|
-
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=
|
108
|
+
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=tZcVSMRJ5-5MoX4ejpXxt5E2suqiQ5AwprQWQiWHhho,85060
|
108
109
|
airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
109
110
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
|
110
|
-
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=
|
111
|
+
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=jVZ3ZV5YZrmDNIX5cM2mugXmnbH27zHRcD22_3oatpo,8454
|
111
112
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
|
112
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
113
|
-
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=
|
113
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=mOO0HahnHP0yv5LHFCayIx98R-yYHw6qkY9T5BxSlBg,98683
|
114
|
+
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=8uGos2u7TFTx_EJBdcjdUGn3Eyx6jUuEa1_VB8UP_dI,631
|
114
115
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
115
116
|
airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=t7pRdFWfFWJtQQG19c9PVeMODyO2BknRTakpM5U9N-8,4844
|
116
117
|
airbyte_cdk/sources/declarative/partition_routers/partition_router.py,sha256=YyEIzdmLd1FjbVP3QbQ2VFCLW_P-OGbVh6VpZShp54k,2218
|
@@ -152,13 +153,10 @@ airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_
|
|
152
153
|
airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py,sha256=8YRiDzjYvqJ-aMmKFcjqzv_-e8OZ5QG_TbpZ-nuCu6s,2590
|
153
154
|
airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
|
154
155
|
airbyte_cdk/sources/declarative/requesters/requester.py,sha256=iVVpXQ4KEd9OyZNwmOofMvx7_06i8ZRxGo3aNTrEQLM,4946
|
155
|
-
airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=8cL--3WCJvtKVIIAqKWoIEhmmf3FemB7PlsnO0Mx_cY,688
|
156
|
-
airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb9artL4ZkeqN8RmEykHH6FJgqXD7fCEnh1X0,1936
|
157
|
-
airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=ZA2vrHQKfXNMcH3x1iuyFOTGNzYDhUFT2qcaiOzSK0A,4271
|
158
156
|
airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=FVQpUGVwp2Gibk4gp07VmLKX5AafUlsZWFSrDpUDuJM,443
|
159
157
|
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=WDFnjrXLz3-YEjFhmlMkWAn9AJvnZ0mk9FyC8DAhEYk,4976
|
160
158
|
airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
|
161
|
-
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=
|
159
|
+
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=6IP6e9cjGEU2y77lcOKj1bqn3bYGBAsP8vJU4Skzp30,24182
|
162
160
|
airbyte_cdk/sources/declarative/schema/__init__.py,sha256=ul8L9S0-__AMEdbCLHBq-PMEeA928NVp8BB83BMotfU,517
|
163
161
|
airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=KTACrIE23a83wsm3Rd9Eb4K6-20lrGqYxTHNp9yxsso,1820
|
164
162
|
airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRsatRJq3R3BeyRR0wIoK3gcP1gcpVRQ_P5U,464
|
@@ -167,7 +165,7 @@ airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLn
|
|
167
165
|
airbyte_cdk/sources/declarative/spec/__init__.py,sha256=H0UwoRhgucbKBIzg85AXrifybVmfpwWpPdy22vZKVuo,141
|
168
166
|
airbyte_cdk/sources/declarative/spec/spec.py,sha256=ODSNUgkDOhnLQnwLjgSaME6R3kNeywjROvbNrWEnsgU,1876
|
169
167
|
airbyte_cdk/sources/declarative/stream_slicers/__init__.py,sha256=sI9vhc95RwJYOnA0VKjcbtKgFcmAbWjhdWBXFbAijOs,176
|
170
|
-
airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=
|
168
|
+
airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=7KE_qBBP3QYA7qQdOE42u3fwUM5S1FD5rowf7gtu3qk,3462
|
171
169
|
airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
|
172
170
|
airbyte_cdk/sources/declarative/transformations/__init__.py,sha256=CPJ8TlMpiUmvG3624VYu_NfTzxwKcfBjM2Q2wJ7fkSA,919
|
173
171
|
airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=r4YdAuAk2bQtNWJMztIIy2CC-NglD9NeK1s1TeO9wkw,5027
|
@@ -248,7 +246,7 @@ airbyte_cdk/sources/streams/concurrent/abstract_stream.py,sha256=3OB5VsvOkJmCxIM
|
|
248
246
|
airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py,sha256=QTry1QCBUwJDw1QSCEvz23s7zIEx_7QMxkPq9j-oPIQ,1358
|
249
247
|
airbyte_cdk/sources/streams/concurrent/adapters.py,sha256=QP_64kQo-b3sRNHZA5aqrgCJqAhIVegRM3vJ8jGyuSY,15213
|
250
248
|
airbyte_cdk/sources/streams/concurrent/availability_strategy.py,sha256=4La5v2UffSjGnhmF4kwNIKt_g3RXk2ux1mSHA1ejgYM,2898
|
251
|
-
airbyte_cdk/sources/streams/concurrent/cursor.py,sha256=
|
249
|
+
airbyte_cdk/sources/streams/concurrent/cursor.py,sha256=SbkWn2t5uxVhT6W657zrENWnxC74oyp_WU9ol-_w5so,21215
|
252
250
|
airbyte_cdk/sources/streams/concurrent/default_stream.py,sha256=K3rLMpYhS7nnmvwQ52lqBy7DQdFMJpvvT7sgBg_ckA8,3207
|
253
251
|
airbyte_cdk/sources/streams/concurrent/exceptions.py,sha256=JOZ446MCLpmF26r9KfS6OO_6rGjcjgJNZdcw6jccjEI,468
|
254
252
|
airbyte_cdk/sources/streams/concurrent/helpers.py,sha256=S6AW8TgIASCZ2UuUcQLE8OzgYUHWt2-KPOvNPwnQf-Q,1596
|
@@ -333,8 +331,8 @@ airbyte_cdk/utils/slice_hasher.py,sha256=-pHexlNYoWYPnXNH-M7HEbjmeJe9Zk7SJijdQ7d
|
|
333
331
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
334
332
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
335
333
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
336
|
-
airbyte_cdk-6.8.
|
337
|
-
airbyte_cdk-6.8.
|
338
|
-
airbyte_cdk-6.8.
|
339
|
-
airbyte_cdk-6.8.
|
340
|
-
airbyte_cdk-6.8.
|
334
|
+
airbyte_cdk-6.8.2.dev1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
335
|
+
airbyte_cdk-6.8.2.dev1.dist-info/METADATA,sha256=yATvM83Zo6tZfb5wnnP-1YGmBjL1ZR2zWzZFr09J1R8,6112
|
336
|
+
airbyte_cdk-6.8.2.dev1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
337
|
+
airbyte_cdk-6.8.2.dev1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
338
|
+
airbyte_cdk-6.8.2.dev1.dist-info/RECORD,,
|
@@ -1,13 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from airbyte_cdk.sources.declarative.resolvers.components_resolver import ComponentsResolver, ComponentMappingDefinition, ResolvedComponentMappingDefinition
|
6
|
-
from airbyte_cdk.sources.declarative.resolvers.http_components_resolver import HttpComponentsResolver
|
7
|
-
from airbyte_cdk.sources.declarative.models import HttpComponentsResolver as HttpComponentsResolverModel
|
8
|
-
|
9
|
-
COMPONENTS_RESOLVER_TYPE_MAPPING = {
|
10
|
-
"HttpComponentsResolver": HttpComponentsResolverModel
|
11
|
-
}
|
12
|
-
|
13
|
-
__all__ = ["ComponentsResolver", "HttpComponentsResolver", "ComponentMappingDefinition", "ResolvedComponentMappingDefinition", "COMPONENTS_RESOLVER_TYPE_MAPPING"]
|
@@ -1,55 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from abc import ABC, abstractmethod
|
6
|
-
from dataclasses import InitVar, dataclass
|
7
|
-
from typing import Any, Dict, Iterable, List, Mapping, Optional, Type, Union
|
8
|
-
|
9
|
-
from typing_extensions import deprecated
|
10
|
-
|
11
|
-
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
12
|
-
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
13
|
-
|
14
|
-
|
15
|
-
@dataclass(frozen=True)
|
16
|
-
class ComponentMappingDefinition:
|
17
|
-
"""Defines the configuration for mapping a component in a stream. This class specifies
|
18
|
-
what field in the stream template should be updated with value, supporting dynamic interpolation
|
19
|
-
and type enforcement."""
|
20
|
-
|
21
|
-
field_path: List["InterpolatedString"]
|
22
|
-
value: Union["InterpolatedString", str]
|
23
|
-
value_type: Optional[Type[Any]]
|
24
|
-
parameters: InitVar[Mapping[str, Any]]
|
25
|
-
|
26
|
-
|
27
|
-
@dataclass(frozen=True)
|
28
|
-
class ResolvedComponentMappingDefinition:
|
29
|
-
"""Defines resolved configuration for mapping a component in a stream. This class specifies
|
30
|
-
what field in the stream template should be updated with value, supporting dynamic interpolation
|
31
|
-
and type enforcement."""
|
32
|
-
|
33
|
-
field_path: List["InterpolatedString"]
|
34
|
-
value: "InterpolatedString"
|
35
|
-
value_type: Optional[Type[Any]]
|
36
|
-
parameters: InitVar[Mapping[str, Any]]
|
37
|
-
|
38
|
-
|
39
|
-
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
40
|
-
@dataclass
|
41
|
-
class ComponentsResolver(ABC):
|
42
|
-
"""
|
43
|
-
Abstract base class for resolving components in a stream template.
|
44
|
-
"""
|
45
|
-
|
46
|
-
@abstractmethod
|
47
|
-
def resolve_components(
|
48
|
-
self, stream_template_config: Dict[str, Any]
|
49
|
-
) -> Iterable[Dict[str, Any]]:
|
50
|
-
"""
|
51
|
-
Maps and populates values into a stream template configuration.
|
52
|
-
:param stream_template_config: The stream template with placeholders for components.
|
53
|
-
:yields: The resolved stream config with populated values.
|
54
|
-
"""
|
55
|
-
pass
|
@@ -1,106 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from copy import deepcopy
|
6
|
-
from dataclasses import InitVar, dataclass, field
|
7
|
-
from typing import Any, Dict, Iterable, List, Mapping
|
8
|
-
|
9
|
-
import dpath
|
10
|
-
from typing_extensions import deprecated
|
11
|
-
|
12
|
-
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
13
|
-
from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
|
14
|
-
ComponentMappingDefinition,
|
15
|
-
ComponentsResolver,
|
16
|
-
ResolvedComponentMappingDefinition,
|
17
|
-
)
|
18
|
-
from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
|
19
|
-
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
20
|
-
from airbyte_cdk.sources.types import Config
|
21
|
-
|
22
|
-
|
23
|
-
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
24
|
-
@dataclass
|
25
|
-
class HttpComponentsResolver(ComponentsResolver):
|
26
|
-
"""
|
27
|
-
Resolves and populates stream templates with components fetched via an HTTP retriever.
|
28
|
-
|
29
|
-
Attributes:
|
30
|
-
retriever (Retriever): The retriever used to fetch data from an API.
|
31
|
-
config (Config): Configuration object for the resolver.
|
32
|
-
components_mapping (List[ComponentMappingDefinition]): List of mappings to resolve.
|
33
|
-
parameters (InitVar[Mapping[str, Any]]): Additional parameters for interpolation.
|
34
|
-
"""
|
35
|
-
|
36
|
-
retriever: Retriever
|
37
|
-
config: Config
|
38
|
-
components_mapping: List[ComponentMappingDefinition]
|
39
|
-
parameters: InitVar[Mapping[str, Any]]
|
40
|
-
_resolved_components: List[ResolvedComponentMappingDefinition] = field(
|
41
|
-
init=False, repr=False, default_factory=list
|
42
|
-
)
|
43
|
-
|
44
|
-
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
45
|
-
"""
|
46
|
-
Initializes and parses component mappings, converting them to resolved definitions.
|
47
|
-
|
48
|
-
Args:
|
49
|
-
parameters (Mapping[str, Any]): Parameters for interpolation.
|
50
|
-
"""
|
51
|
-
for component_mapping in self.components_mapping:
|
52
|
-
if isinstance(component_mapping.value, (str, InterpolatedString)):
|
53
|
-
interpolated_value = (
|
54
|
-
InterpolatedString.create(component_mapping.value, parameters=parameters)
|
55
|
-
if isinstance(component_mapping.value, str)
|
56
|
-
else component_mapping.value
|
57
|
-
)
|
58
|
-
|
59
|
-
field_path = [
|
60
|
-
InterpolatedString.create(path, parameters=parameters)
|
61
|
-
for path in component_mapping.field_path
|
62
|
-
]
|
63
|
-
|
64
|
-
self._resolved_components.append(
|
65
|
-
ResolvedComponentMappingDefinition(
|
66
|
-
field_path=field_path,
|
67
|
-
value=interpolated_value,
|
68
|
-
value_type=component_mapping.value_type,
|
69
|
-
parameters=parameters,
|
70
|
-
)
|
71
|
-
)
|
72
|
-
else:
|
73
|
-
raise ValueError(
|
74
|
-
f"Expected a string or InterpolatedString for value in mapping: {component_mapping}"
|
75
|
-
)
|
76
|
-
|
77
|
-
def resolve_components(
|
78
|
-
self, stream_template_config: Dict[str, Any]
|
79
|
-
) -> Iterable[Dict[str, Any]]:
|
80
|
-
"""
|
81
|
-
Resolves components in the stream template configuration by populating values.
|
82
|
-
|
83
|
-
Args:
|
84
|
-
stream_template_config (Dict[str, Any]): Stream template to populate.
|
85
|
-
|
86
|
-
Yields:
|
87
|
-
Dict[str, Any]: Updated configurations with resolved components.
|
88
|
-
"""
|
89
|
-
kwargs = {"stream_template_config": stream_template_config}
|
90
|
-
|
91
|
-
for components_values in self.retriever.read_records({}):
|
92
|
-
updated_config = deepcopy(stream_template_config)
|
93
|
-
kwargs["components_values"] = components_values # type: ignore[assignment] # component_values will always be of type Mapping[str, Any]
|
94
|
-
|
95
|
-
for resolved_component in self._resolved_components:
|
96
|
-
valid_types = (
|
97
|
-
(resolved_component.value_type,) if resolved_component.value_type else None
|
98
|
-
)
|
99
|
-
value = resolved_component.value.eval(
|
100
|
-
self.config, valid_types=valid_types, **kwargs
|
101
|
-
)
|
102
|
-
|
103
|
-
path = [path.eval(self.config, **kwargs) for path in resolved_component.field_path]
|
104
|
-
dpath.set(updated_config, path, value)
|
105
|
-
|
106
|
-
yield updated_config
|
File without changes
|
File without changes
|
File without changes
|