airbyte-cdk 6.43.0.dev0__py3-none-any.whl → 6.44.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/connector_builder/connector_builder_handler.py +22 -8
- airbyte_cdk/connector_builder/main.py +3 -3
- airbyte_cdk/sources/declarative/checks/__init__.py +5 -2
- airbyte_cdk/sources/declarative/checks/check_stream.py +113 -11
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +23 -129
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +6 -0
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +21 -87
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +16 -4
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +44 -192
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +2 -25
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +31 -100
- airbyte_cdk/sources/declarative/transformations/add_fields.py +3 -1
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/RECORD +18 -23
- airbyte_cdk/sources/declarative/requesters/query_properties/__init__.py +0 -14
- airbyte_cdk/sources/declarative/requesters/query_properties/group_by_key.py +0 -24
- airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py +0 -40
- airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py +0 -65
- airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py +0 -48
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/entry_points.txt +0 -0
@@ -7,8 +7,8 @@ airbyte_cdk/config_observation.py,sha256=7SSPxtN0nXPkm4euGNcTTr1iLbwUL01jy-24V1H
|
|
7
7
|
airbyte_cdk/connector.py,sha256=bO23kdGRkl8XKFytOgrrWFc_VagteTHVEF6IsbizVkM,4224
|
8
8
|
airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz52ftyNRnBw,1665
|
9
9
|
airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
10
|
-
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=
|
11
|
-
airbyte_cdk/connector_builder/main.py,sha256=
|
10
|
+
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=gwFpZHmvFrMB9NMlRuRfq0KEkbPaFucCB8k1DL20Wmw,5769
|
11
|
+
airbyte_cdk/connector_builder/main.py,sha256=7hE4GG8yTLSas_nUuANsPJQVBZyOK-l_1Q0vf0FjY5o,3839
|
12
12
|
airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
|
13
13
|
airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
|
14
14
|
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=Iczn-_iczS2CaIAunWwyFcX0uLTra8Wh9JVfzm1Gfxo,26765
|
@@ -61,9 +61,9 @@ airbyte_cdk/sources/declarative/auth/oauth.py,sha256=SUfib1oSzlyRRnOSg8Bui73mfyr
|
|
61
61
|
airbyte_cdk/sources/declarative/auth/selective_authenticator.py,sha256=qGwC6YsCldr1bIeKG6Qo-A9a5cTdHw-vcOn3OtQrS4c,1540
|
62
62
|
airbyte_cdk/sources/declarative/auth/token.py,sha256=2EnE78EhBOY9hbeZnQJ9AuFaM-G7dccU-oKo_LThRQk,11070
|
63
63
|
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=Jzuxlmt1_-_aFC_n0OmP8L1nDOacLzbEVVx3kjdX_W8,3104
|
64
|
-
airbyte_cdk/sources/declarative/checks/__init__.py,sha256=
|
64
|
+
airbyte_cdk/sources/declarative/checks/__init__.py,sha256=cpoBwEbRNcMi7Rmi5pD63ppUvAOZsAWzasmc57ob9rc,873
|
65
65
|
airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py,sha256=HUktywjI8pqOeED08UGqponUSwxs2TOAECTowlWlrRE,2138
|
66
|
-
airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=
|
66
|
+
airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=QeExVmpSYjr_CnghHuJBn5oHW6wyKmAr7rotSSfcWp8,7145
|
67
67
|
airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
|
68
68
|
airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
|
69
69
|
airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
|
@@ -71,7 +71,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=uhy0dRkA
|
|
71
71
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
72
72
|
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
|
73
73
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
74
|
-
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256
|
74
|
+
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=pMzxm5YCtLM30wPIniTQk39oFRzvzsq7R_Mj2KEmaSg,153851
|
75
75
|
airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
|
76
76
|
airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
|
77
77
|
airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
|
@@ -109,18 +109,18 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkH
|
|
109
109
|
airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
|
110
110
|
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
|
111
111
|
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=CF2L_r73TCd1Wssq9_tjFPb4c872QM8oW6NL-DIR0h4,5226
|
112
|
-
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=
|
112
|
+
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=tUe54Xgy7dq-a7HNaPogk6AXlVeqQXHWde6_8_Xy0ow,19130
|
113
113
|
airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
114
114
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
|
115
115
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
116
116
|
airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
|
117
|
-
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=
|
117
|
+
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=7lR9dulMMnCJps7hpbuN4Moh9FFG8sGaEIcsK4zi_7s,108868
|
118
118
|
airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
119
119
|
airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
|
120
120
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
|
121
|
-
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=
|
121
|
+
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=4C15MKV-zOrMVQAm4FyohDsrJUBCSpMv5tZw0SK3aeI,9685
|
122
122
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
|
123
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
123
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=9nlof-3xlaYC_WxOFJ5gj065DCAIxkUo1Re5FOPiTnM,151021
|
124
124
|
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
|
125
125
|
airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
|
126
126
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
@@ -156,18 +156,13 @@ airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_incremen
|
|
156
156
|
airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py,sha256=Z2i6a-oKMmOTxHxsTVSnyaShkJ3u8xZw1xIJdx2yxss,2731
|
157
157
|
airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py,sha256=ZBshGQNr5Bb_V8dqnWRISqdXFcjm1CKIXnlfbRhNl8g,1308
|
158
158
|
airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py,sha256=LoKXdUbSgHEtSwtA8DFrnX6SpQbRVVwreY8NguTKTcI,2229
|
159
|
-
airbyte_cdk/sources/declarative/requesters/query_properties/__init__.py,sha256=6Da7wsYQxzWWgpkisj47YbMPtj1sgg1b_60qTspO_Pw,604
|
160
|
-
airbyte_cdk/sources/declarative/requesters/query_properties/group_by_key.py,sha256=7uVXFwzR5Q5ngkwAtW8x3QBK963Ub1Qpo2vapF8IIc0,659
|
161
|
-
airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py,sha256=Rz4CViWqDGKCCZ8eYbb5yVlF-ElXmBmMZqhYglFxGqA,1395
|
162
|
-
airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py,sha256=o9Jq-7mBC8ahPii5EngA0h4gBB8UV0xxZbifG9otrIk,2254
|
163
|
-
airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py,sha256=6xULj0wzHzc8l6jV-v-fOtbeli8peBfvyeSLtioN0-0,1672
|
164
159
|
airbyte_cdk/sources/declarative/requesters/request_option.py,sha256=Bl0gxGWudmwT3FXBozTN00WYle2jd6ry_S1YylCnwqM,4825
|
165
160
|
airbyte_cdk/sources/declarative/requesters/request_options/__init__.py,sha256=WCwpKqM4wKqy-DHJaCHbKAlFqRVOqMi9K5qonxIfi_Y,809
|
166
161
|
airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py,sha256=31nG6_0igidJFQon37-WeQkTpG3g2A5ZmlluI3ilZdE,3632
|
167
162
|
airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py,sha256=SRROdPJZ5kuqHLOlkh115pWP9nDGfDxRYPgH9oD3hPo,1798
|
168
163
|
airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py,sha256=86YozYuBDfu0t9NbevIvQoGU0vqTP4rt3dRSTsHz3PA,2269
|
169
164
|
airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py,sha256=rR00kE64U2yL0McU1gPr4_W5_sLUqwDgL3Nvj691nRU,2884
|
170
|
-
airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py,sha256=
|
165
|
+
airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py,sha256=vOsdHfWHiTFc89WENHPv1hcxLgdzycMXVT_IEtLuhfs,5012
|
171
166
|
airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py,sha256=8YRiDzjYvqJ-aMmKFcjqzv_-e8OZ5QG_TbpZ-nuCu6s,2590
|
172
167
|
airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
|
173
168
|
airbyte_cdk/sources/declarative/requesters/requester.py,sha256=OcDzuCBgD1owK_lBPG0KbRRHRn9kzbuRveU4HejDiv4,5116
|
@@ -178,7 +173,7 @@ airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=Aio
|
|
178
173
|
airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=nQepwG_RfW53sgwvK5dLPqfCx0VjsQ83nYoPjBMAaLM,527
|
179
174
|
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=6oZtnCHm9NdDvjTSrVwPQOXGSdETSIR7eWH2vFjM7jI,4855
|
180
175
|
airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
|
181
|
-
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=
|
176
|
+
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=p6O4FYS7zzPq6uQT2NVnughUjI66tePaXVlyhCAyyv0,27746
|
182
177
|
airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
|
183
178
|
airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=KTACrIE23a83wsm3Rd9Eb4K6-20lrGqYxTHNp9yxsso,1820
|
184
179
|
airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=J8Q_iJYhcSQLWyt0bTZCbDAGpxt9G8FCc6Q9jtGsNzw,10703
|
@@ -191,7 +186,7 @@ airbyte_cdk/sources/declarative/stream_slicers/__init__.py,sha256=sI9vhc95RwJYOn
|
|
191
186
|
airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=RW1Q44ml-VWeMl4lNcV6EfyzrzCZkjj-hd0Omx_n_n4,3405
|
192
187
|
airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
|
193
188
|
airbyte_cdk/sources/declarative/transformations/__init__.py,sha256=CPJ8TlMpiUmvG3624VYu_NfTzxwKcfBjM2Q2wJ7fkSA,919
|
194
|
-
airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=
|
189
|
+
airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=Eg1jQtRObgzxbtySTQs5uEZIjEklsoHFxYSPf78x6Ng,5420
|
195
190
|
airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py,sha256=I8oXPAOFhBV1mW_ufMn8Ii7oMbtect0sfLcpBNrKzzw,2374
|
196
191
|
airbyte_cdk/sources/declarative/transformations/flatten_fields.py,sha256=yT3owG6rMKaRX-LJ_T-jSTnh1B5NoAHyH4YZN9yOvE8,1758
|
197
192
|
airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py,sha256=vbIn6ump-Ut6g20yMub7PFoPBhOKVtrHSAUdcOUdLfw,1999
|
@@ -364,9 +359,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
364
359
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
365
360
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
366
361
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
367
|
-
airbyte_cdk-6.
|
368
|
-
airbyte_cdk-6.
|
369
|
-
airbyte_cdk-6.
|
370
|
-
airbyte_cdk-6.
|
371
|
-
airbyte_cdk-6.
|
372
|
-
airbyte_cdk-6.
|
362
|
+
airbyte_cdk-6.44.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
363
|
+
airbyte_cdk-6.44.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
364
|
+
airbyte_cdk-6.44.0.dist-info/METADATA,sha256=kV2dhymFTY01knD0E2Ep9U3mb92lVtiEOAix38sgyCI,6071
|
365
|
+
airbyte_cdk-6.44.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
366
|
+
airbyte_cdk-6.44.0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
367
|
+
airbyte_cdk-6.44.0.dist-info/RECORD,,
|
@@ -1,14 +0,0 @@
|
|
1
|
-
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
2
|
-
|
3
|
-
from airbyte_cdk.sources.declarative.requesters.query_properties.group_by_key import GroupByKey
|
4
|
-
from airbyte_cdk.sources.declarative.requesters.query_properties.properties_from_endpoint import (
|
5
|
-
PropertiesFromEndpoint,
|
6
|
-
)
|
7
|
-
from airbyte_cdk.sources.declarative.requesters.query_properties.property_chunking import (
|
8
|
-
PropertyChunking,
|
9
|
-
)
|
10
|
-
from airbyte_cdk.sources.declarative.requesters.query_properties.query_properties import (
|
11
|
-
QueryProperties,
|
12
|
-
)
|
13
|
-
|
14
|
-
__all__ = ["GroupByKey", "PropertiesFromEndpoint", "PropertyChunking", "QueryProperties"]
|
@@ -1,24 +0,0 @@
|
|
1
|
-
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
2
|
-
|
3
|
-
from dataclasses import InitVar, dataclass
|
4
|
-
from typing import Any, List, Mapping, Union
|
5
|
-
|
6
|
-
from airbyte_cdk.sources.types import Config, Record
|
7
|
-
|
8
|
-
|
9
|
-
@dataclass
|
10
|
-
class GroupByKey:
|
11
|
-
"""
|
12
|
-
tbd
|
13
|
-
"""
|
14
|
-
|
15
|
-
key: Union[str, List[str]]
|
16
|
-
parameters: InitVar[Mapping[str, Any]]
|
17
|
-
config: Config
|
18
|
-
|
19
|
-
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
20
|
-
self._keys = [self.key] if isinstance(self.key, str) else self.key
|
21
|
-
|
22
|
-
def get_group_key(self, record: Record) -> str:
|
23
|
-
resolved_keys = [str(record.data.get(key)) for key in self._keys]
|
24
|
-
return ",".join(resolved_keys)
|
@@ -1,40 +0,0 @@
|
|
1
|
-
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
2
|
-
|
3
|
-
from dataclasses import InitVar, dataclass
|
4
|
-
from typing import Any, Iterable, List, Mapping, Optional, Union
|
5
|
-
|
6
|
-
import dpath
|
7
|
-
|
8
|
-
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
9
|
-
from airbyte_cdk.sources.declarative.retrievers import Retriever
|
10
|
-
from airbyte_cdk.sources.types import Config, StreamSlice
|
11
|
-
|
12
|
-
|
13
|
-
@dataclass
|
14
|
-
class PropertiesFromEndpoint:
|
15
|
-
"""
|
16
|
-
tbd
|
17
|
-
"""
|
18
|
-
|
19
|
-
property_field_path: List[str]
|
20
|
-
retriever: Retriever
|
21
|
-
config: Config
|
22
|
-
parameters: InitVar[Mapping[str, Any]]
|
23
|
-
|
24
|
-
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
25
|
-
self._property_field_path = [
|
26
|
-
InterpolatedString(string=property_field, parameters=parameters)
|
27
|
-
for property_field in self.property_field_path
|
28
|
-
]
|
29
|
-
|
30
|
-
def get_properties_from_endpoint(self, stream_slice: Optional[StreamSlice]) -> Iterable[str]:
|
31
|
-
response_properties = self.retriever.read_records(
|
32
|
-
records_schema={}, stream_slice=stream_slice
|
33
|
-
)
|
34
|
-
for property_obj in response_properties:
|
35
|
-
path = [
|
36
|
-
node.eval(self.config) if not isinstance(node, str) else node
|
37
|
-
for node in self._property_field_path
|
38
|
-
]
|
39
|
-
|
40
|
-
yield dpath.get(property_obj, path) # type: ignore # extracted will be a MutableMapping, given input data structure
|
@@ -1,65 +0,0 @@
|
|
1
|
-
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
2
|
-
|
3
|
-
from dataclasses import InitVar, dataclass
|
4
|
-
from enum import Enum
|
5
|
-
from typing import Any, Iterable, List, Mapping, Optional
|
6
|
-
|
7
|
-
from airbyte_cdk.sources.declarative.requesters.query_properties import GroupByKey
|
8
|
-
from airbyte_cdk.sources.types import Config, Record
|
9
|
-
|
10
|
-
|
11
|
-
class PropertyLimitType(Enum):
|
12
|
-
"""
|
13
|
-
yeah
|
14
|
-
"""
|
15
|
-
|
16
|
-
characters = "characters"
|
17
|
-
property_count = "property_count"
|
18
|
-
|
19
|
-
|
20
|
-
@dataclass
|
21
|
-
class PropertyChunking:
|
22
|
-
"""
|
23
|
-
tbd
|
24
|
-
"""
|
25
|
-
|
26
|
-
property_limit_type: PropertyLimitType
|
27
|
-
property_limit: Optional[int]
|
28
|
-
record_merge_strategy: Optional[
|
29
|
-
GroupByKey
|
30
|
-
] # This should eventually be some sort of interface or type
|
31
|
-
parameters: InitVar[Mapping[str, Any]]
|
32
|
-
config: Config
|
33
|
-
|
34
|
-
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
35
|
-
self._record_merge_strategy = self.record_merge_strategy or GroupByKey(
|
36
|
-
key="id", config=self.config, parameters=parameters
|
37
|
-
)
|
38
|
-
|
39
|
-
def get_request_property_chunks(
|
40
|
-
self, property_fields: Iterable[str], always_include_properties: Optional[List[str]]
|
41
|
-
) -> Iterable[List[str]]:
|
42
|
-
if not self.property_limit:
|
43
|
-
single_property_chunk = list(property_fields)
|
44
|
-
if always_include_properties:
|
45
|
-
single_property_chunk.extend(always_include_properties)
|
46
|
-
yield single_property_chunk
|
47
|
-
return
|
48
|
-
current_chunk = list(always_include_properties) if always_include_properties else []
|
49
|
-
chunk_size = 0
|
50
|
-
for property_field in property_fields:
|
51
|
-
property_field_size = (
|
52
|
-
len(property_field)
|
53
|
-
if self.property_limit_type == PropertyLimitType.characters
|
54
|
-
else 1
|
55
|
-
)
|
56
|
-
if chunk_size + property_field_size > self.property_limit:
|
57
|
-
yield current_chunk
|
58
|
-
current_chunk = list(always_include_properties) if always_include_properties else []
|
59
|
-
chunk_size = 0
|
60
|
-
current_chunk.append(property_field)
|
61
|
-
chunk_size += property_field_size
|
62
|
-
yield current_chunk
|
63
|
-
|
64
|
-
def get_merge_key(self, record: Record) -> str:
|
65
|
-
return self._record_merge_strategy.get_group_key(record=record)
|
@@ -1,48 +0,0 @@
|
|
1
|
-
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
2
|
-
|
3
|
-
from dataclasses import InitVar, dataclass
|
4
|
-
from typing import Any, Iterable, List, Mapping, Optional, Union
|
5
|
-
|
6
|
-
from airbyte_cdk.sources.declarative.requesters.query_properties import (
|
7
|
-
PropertiesFromEndpoint,
|
8
|
-
PropertyChunking,
|
9
|
-
)
|
10
|
-
from airbyte_cdk.sources.types import Config, StreamSlice
|
11
|
-
|
12
|
-
|
13
|
-
@dataclass
|
14
|
-
class QueryProperties:
|
15
|
-
"""
|
16
|
-
tbd
|
17
|
-
"""
|
18
|
-
|
19
|
-
property_list: Optional[Union[List[str], PropertiesFromEndpoint]]
|
20
|
-
always_include_properties: Optional[List[str]]
|
21
|
-
property_chunking: Optional[PropertyChunking]
|
22
|
-
config: Config
|
23
|
-
parameters: InitVar[Mapping[str, Any]]
|
24
|
-
|
25
|
-
def get_request_property_chunks(
|
26
|
-
self, stream_slice: Optional[StreamSlice] = None
|
27
|
-
) -> Iterable[List[str]]:
|
28
|
-
fields: Union[Iterable[str], List[str]]
|
29
|
-
if isinstance(self.property_list, PropertiesFromEndpoint):
|
30
|
-
fields = self.property_list.get_properties_from_endpoint(stream_slice=stream_slice)
|
31
|
-
else:
|
32
|
-
fields = self.property_list if self.property_list else []
|
33
|
-
|
34
|
-
if self.property_chunking:
|
35
|
-
yield from self.property_chunking.get_request_property_chunks(
|
36
|
-
property_fields=fields, always_include_properties=self.always_include_properties
|
37
|
-
)
|
38
|
-
else:
|
39
|
-
yield from [list(fields)]
|
40
|
-
|
41
|
-
def has_multiple_chunks(self, stream_slice: Optional[StreamSlice]) -> bool:
|
42
|
-
property_chunks = iter(self.get_request_property_chunks(stream_slice=stream_slice))
|
43
|
-
try:
|
44
|
-
next(property_chunks)
|
45
|
-
next(property_chunks)
|
46
|
-
return True
|
47
|
-
except StopIteration:
|
48
|
-
return False
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|