airbyte-cdk 6.37.0.dev1__py3-none-any.whl → 6.37.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/connector_builder/models.py +16 -14
- airbyte_cdk/connector_builder/test_reader/helpers.py +120 -22
- airbyte_cdk/connector_builder/test_reader/message_grouper.py +16 -3
- airbyte_cdk/connector_builder/test_reader/types.py +9 -1
- airbyte_cdk/sources/declarative/auth/token_provider.py +1 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +43 -7
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +7 -1
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +77 -48
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +13 -2
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +83 -17
- airbyte_cdk/sources/declarative/interpolation/macros.py +2 -0
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +37 -50
- airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +18 -4
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +171 -70
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +0 -4
- airbyte_cdk/sources/declarative/requesters/README.md +5 -5
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +60 -17
- airbyte_cdk/sources/declarative/requesters/http_requester.py +49 -17
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +25 -4
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +6 -1
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +7 -2
- airbyte_cdk/sources/declarative/requesters/requester.py +7 -1
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +10 -3
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +21 -4
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +2 -2
- airbyte_cdk/sources/http_logger.py +3 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +2 -1
- airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +3 -3
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -0
- airbyte_cdk/sources/types.py +1 -0
- airbyte_cdk/utils/mapping_helpers.py +18 -1
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/METADATA +4 -4
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/RECORD +39 -44
- airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py +0 -136
- airbyte_cdk/sources/embedded/__init__.py +0 -3
- airbyte_cdk/sources/embedded/base_integration.py +0 -61
- airbyte_cdk/sources/embedded/catalog.py +0 -57
- airbyte_cdk/sources/embedded/runner.py +0 -57
- airbyte_cdk/sources/embedded/tools.py +0 -27
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/entry_points.txt +0 -0
@@ -9,12 +9,12 @@ airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz
|
|
9
9
|
airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
10
10
|
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=BntqkP63RBPvGCtB3CrLLtYplfSlBR42kwXyyk4YGas,4268
|
11
11
|
airbyte_cdk/connector_builder/main.py,sha256=ubAPE0Oo5gjZOa-KMtLLJQkc8_inUpFR3sIb2DEh2No,3722
|
12
|
-
airbyte_cdk/connector_builder/models.py,sha256=
|
12
|
+
airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
|
13
13
|
airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
|
14
|
-
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=
|
15
|
-
airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=
|
14
|
+
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=Iczn-_iczS2CaIAunWwyFcX0uLTra8Wh9JVfzm1Gfxo,26765
|
15
|
+
airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=84BAEPIBHMq3WCfO14WNvh_q7OsjGgDt0q1FTu8eW-w,6918
|
16
16
|
airbyte_cdk/connector_builder/test_reader/reader.py,sha256=GurMB4ITO_PntvhIHSJkXbhynLilI4DObY5A2axavXo,20667
|
17
|
-
airbyte_cdk/connector_builder/test_reader/types.py,sha256=
|
17
|
+
airbyte_cdk/connector_builder/test_reader/types.py,sha256=hPZG3jO03kBaPyW94NI3JHRS1jxXGSNBcN1HFzOxo5Y,2528
|
18
18
|
airbyte_cdk/destinations/__init__.py,sha256=FyDp28PT_YceJD5HDFhA-mrGfX9AONIyMQ4d68CHNxQ,213
|
19
19
|
airbyte_cdk/destinations/destination.py,sha256=CIq-yb8C_0QvcKCtmStaHfiqn53GEfRAIGGCkJhKP1Q,5880
|
20
20
|
airbyte_cdk/destinations/vector_db_based/README.md,sha256=QAe8c_1Afme4r2TCE10cTSaxUE3zgCBuArSuRQqK8tA,2115
|
@@ -60,22 +60,22 @@ airbyte_cdk/sources/declarative/auth/jwt.py,sha256=SICqNsN2Cn_EgKadIgWuZpQxuMHyz
|
|
60
60
|
airbyte_cdk/sources/declarative/auth/oauth.py,sha256=SUfib1oSzlyRRnOSg8Bui73mfyrcyr9OssdchbKdu4s,14162
|
61
61
|
airbyte_cdk/sources/declarative/auth/selective_authenticator.py,sha256=qGwC6YsCldr1bIeKG6Qo-A9a5cTdHw-vcOn3OtQrS4c,1540
|
62
62
|
airbyte_cdk/sources/declarative/auth/token.py,sha256=2EnE78EhBOY9hbeZnQJ9AuFaM-G7dccU-oKo_LThRQk,11070
|
63
|
-
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=
|
63
|
+
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=Jzuxlmt1_-_aFC_n0OmP8L1nDOacLzbEVVx3kjdX_W8,3104
|
64
64
|
airbyte_cdk/sources/declarative/checks/__init__.py,sha256=nsVV5Bo0E_tBNd8A4Xdsdb-75PpcLo5RQu2RQ_Gv-ME,806
|
65
65
|
airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py,sha256=HUktywjI8pqOeED08UGqponUSwxs2TOAECTowlWlrRE,2138
|
66
66
|
airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQrilWCfJmncBzXCZ18ptRNip3XA,2139
|
67
67
|
airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
|
68
68
|
airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
|
69
69
|
airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
|
70
|
-
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=
|
70
|
+
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=rAp-sgld4n8Tmybz-51m7VcYXqKwzKDpCJVr1elmkRc,26824
|
71
71
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
|
72
|
-
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=
|
72
|
+
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=0qs4hhmh_XOy2B4MHCn2qVMM79C6MizIBqnvpZj1aSE,2923
|
73
73
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
74
|
-
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=
|
74
|
+
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=n8hJVquDj00_VS_I0B2QgwYNcNcfsVZdkajAKArcOHU,147487
|
75
75
|
airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
|
76
76
|
airbyte_cdk/sources/declarative/declarative_stream.py,sha256=venZjfpvtqr3oFSuvMBWtn4h9ayLhD4L65ACuXCDZ64,10445
|
77
77
|
airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
|
78
|
-
airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=
|
78
|
+
airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=DJbWaaJ5LHCBpyWz-4bEw8rqtJYqabEYZtxnfRtWFE0,4946
|
79
79
|
airbyte_cdk/sources/declarative/decoders/decoder.py,sha256=sl-Gt8lXi7yD2Q-sD8je5QS2PbgrgsYjxRLWsay7DMc,826
|
80
80
|
airbyte_cdk/sources/declarative/decoders/json_decoder.py,sha256=BdWpXXPhEGf_zknggJmhojLosmxuw51RBVTS0jvdCPc,2080
|
81
81
|
airbyte_cdk/sources/declarative/decoders/noop_decoder.py,sha256=iZh0yKY_JzgBnJWiubEusf5c0o6Khd-8EWFWT-8EgFo,542
|
@@ -89,10 +89,10 @@ airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8V
|
|
89
89
|
airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
|
90
90
|
airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
|
91
91
|
airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=HCqx7IyENM_aRF4it2zJN26_vDu6WeP8XgCxQWHUvcY,6934
|
92
|
-
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=
|
92
|
+
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=L6hQV7bdwEp8y-TBMeQY-xmrNyRggL14lKXdWnzYFfA,6622
|
93
93
|
airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
|
94
94
|
airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
|
95
|
-
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=
|
95
|
+
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=MT5JbdEbnPzk3VWZGGvThe4opoX5dHhSXFrnTRYC6dg,22210
|
96
96
|
airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
|
97
97
|
airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
|
98
98
|
airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
|
@@ -107,28 +107,27 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha
|
|
107
107
|
airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkHqGlfa87G6VYMtBAQWin7ECKpfMdrDcg0JO5_rhc,3212
|
108
108
|
airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
|
109
109
|
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
|
110
|
-
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=
|
110
|
+
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=uuXBZUWDWM-sPcUKjNSPRN657QhNQCx_hnhTuJj2zOA,5129
|
111
111
|
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=TN6GCgLXaWDONTaJwQ3A5ELqC-sxwKz-UYSraJYB-dI,17078
|
112
112
|
airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
113
113
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
|
114
114
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
115
115
|
airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
|
116
|
-
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=
|
116
|
+
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=DfbPi512ovaBSWDICJfjIkC3pXDn2aNr1BP-eiLOLyA,103556
|
117
117
|
airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
118
|
-
airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=
|
118
|
+
airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=jDw_TttD3_hpfevXOH-0Ws0eRuqt6wvED0BqosGPRjI,5938
|
119
119
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
|
120
120
|
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
|
121
121
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
|
122
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
123
|
-
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=
|
122
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Mx0KJGbqIZeUWduKy-UvpVH-DRm0pzXDcz203r69oNY,140619
|
123
|
+
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
|
124
124
|
airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
|
125
125
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
126
|
-
airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py,sha256=PFoY931wC1i7Elphrd7LCFUPYKOTPEovLXC-mvkQow0,5531
|
127
126
|
airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=tmGGpMoOBmaMfhVZq53AEWxoHm2lmNVi6hA2_IVEnAA,4882
|
128
127
|
airbyte_cdk/sources/declarative/partition_routers/partition_router.py,sha256=YyEIzdmLd1FjbVP3QbQ2VFCLW_P-OGbVh6VpZShp54k,2218
|
129
128
|
airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py,sha256=SKzKjSyfccq4dxGIh-J6ejrgkCHzaiTIazmbmeQiRD4,1942
|
130
129
|
airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py,sha256=LlWj-Ofs-xfjlqmDzH8OYpyblP2Pb8bPDdR9g1UZyt0,17693
|
131
|
-
airbyte_cdk/sources/declarative/requesters/README.md,sha256=
|
130
|
+
airbyte_cdk/sources/declarative/requesters/README.md,sha256=DQll2qsIzzTiiP35kJp16ONpr7cFeUQNgPfhl5krB24,2675
|
132
131
|
airbyte_cdk/sources/declarative/requesters/__init__.py,sha256=d7a3OoHbqaJDyyPli3nqqJ2yAW_SLX6XDaBAKOwvpxw,364
|
133
132
|
airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py,sha256=SkEDcJxlT1683rNx93K9whoS0OyUukkuOfToGtgpF58,776
|
134
133
|
airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py,sha256=1WZdpFmWL6W_Dko0qjflTaKIWeqt8jHT-D6HcujIp3s,884
|
@@ -143,12 +142,12 @@ airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.
|
|
143
142
|
airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py,sha256=q0YkeYUUWO6iErUy0vjqiOkhg8_9d5YcCmtlpXAJJ9E,1314
|
144
143
|
airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py,sha256=Tan66odx8VHzfdyyXMQkXz2pJYksllGqvxmpoajgcK4,669
|
145
144
|
airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py,sha256=E-fQbt4ShfxZVoqfnmOx69C6FUPWZz8BIqI3DN9Kcjs,7935
|
146
|
-
airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=
|
147
|
-
airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=
|
145
|
+
airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=4wpP0ZNTMLugi-Rc1OFdFaxWfRZSl45nzhHqMFCE8SQ,11924
|
146
|
+
airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=KS2nEUKMnyow8HvvfJJbWdPjlglIWSOZIAxrwkmXCI4,16356
|
148
147
|
airbyte_cdk/sources/declarative/requesters/paginators/__init__.py,sha256=uArbKs9JKNCt7t9tZoeWwjDpyI1HoPp29FNW0JzvaEM,644
|
149
|
-
airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=
|
150
|
-
airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=
|
151
|
-
airbyte_cdk/sources/declarative/requesters/paginators/paginator.py,sha256=
|
148
|
+
airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=SB-Af3CRb4mJwhm4EKNxzl_PK2w5QS4tqrSNNMO2IV4,12760
|
149
|
+
airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=b1-zKxYOUMHn7ahdWpzKEzfG4A7s_WQWy-vzRqZWzME,2152
|
150
|
+
airbyte_cdk/sources/declarative/requesters/paginators/paginator.py,sha256=TzJF1Q-CFlsHF9lMSfmnGCxRYm9_UQCmBcHYQpc7F30,2376
|
152
151
|
airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py,sha256=2gly8fuZpDNwtu1Qg6oE2jBLGqQRdzSLJdnpk_iDV6I,767
|
153
152
|
airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py,sha256=yLzzK5YIRTkXd2Z-BS__AZXuTd6HXjJIxq05K-lQoxI,3898
|
154
153
|
airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py,sha256=WvGt_DTFcAgTR-NHrlrR7B71yG-L6jmfW-Gwm9iYzjY,3624
|
@@ -164,15 +163,15 @@ airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_
|
|
164
163
|
airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py,sha256=vOsdHfWHiTFc89WENHPv1hcxLgdzycMXVT_IEtLuhfs,5012
|
165
164
|
airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py,sha256=8YRiDzjYvqJ-aMmKFcjqzv_-e8OZ5QG_TbpZ-nuCu6s,2590
|
166
165
|
airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
|
167
|
-
airbyte_cdk/sources/declarative/requesters/requester.py,sha256=
|
166
|
+
airbyte_cdk/sources/declarative/requesters/requester.py,sha256=OcDzuCBgD1owK_lBPG0KbRRHRn9kzbuRveU4HejDiv4,5116
|
168
167
|
airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=NiDcz5qi8HPsfX94MUmnX0Rgs_kQXGvucOmJjNWlxKQ,1207
|
169
168
|
airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb9artL4ZkeqN8RmEykHH6FJgqXD7fCEnh1X0,1936
|
170
169
|
airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
|
171
170
|
airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
|
172
171
|
airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=ix9m1dkR69DcXCXUKC5RK_ZZM7ojTLBQ4IkWQTfmfCk,456
|
173
|
-
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=
|
172
|
+
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=dwYZ70eg9DKHEqZydHhMFPkEILbNcXu7E-djOCikNgI,3530
|
174
173
|
airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
|
175
|
-
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=
|
174
|
+
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=fDhc6dMx75UImh1_TfLm4Le59tsHpqIUZnau7uIJyYw,25043
|
176
175
|
airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
|
177
176
|
airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=KTACrIE23a83wsm3Rd9Eb4K6-20lrGqYxTHNp9yxsso,1820
|
178
177
|
airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=J8Q_iJYhcSQLWyt0bTZCbDAGpxt9G8FCc6Q9jtGsNzw,10703
|
@@ -190,16 +189,11 @@ airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py,sha256=1
|
|
190
189
|
airbyte_cdk/sources/declarative/transformations/flatten_fields.py,sha256=yT3owG6rMKaRX-LJ_T-jSTnh1B5NoAHyH4YZN9yOvE8,1758
|
191
190
|
airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py,sha256=vbIn6ump-Ut6g20yMub7PFoPBhOKVtrHSAUdcOUdLfw,1999
|
192
191
|
airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py,sha256=RTs5KX4V3hM7A6QN1WlGF21YccTIyNH6qQI9IMb__hw,670
|
193
|
-
airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py,sha256=
|
192
|
+
airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py,sha256=_3ldEbsA7tQK-zzeU_cG86D1_1SY3wAo1vHE0zXrOck,2265
|
194
193
|
airbyte_cdk/sources/declarative/transformations/remove_fields.py,sha256=EwUP0SZ2p4GRJ6Q8CUzlz9dcUeEidEFDlI2IBye2tlc,2745
|
195
194
|
airbyte_cdk/sources/declarative/transformations/transformation.py,sha256=4sXtx9cNY2EHUPq-xHvDs8GQEBUy3Eo6TkRLKHPXx68,1161
|
196
195
|
airbyte_cdk/sources/declarative/types.py,sha256=yqx0xlZv_76tkC7fqJKefmvl4GJJ8mXbeddwVV8XRJU,778
|
197
196
|
airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256=MsKSAqtpwIqJfYOiUX01RbqMeTy7pvBoguvyTWrL7pI,2390
|
198
|
-
airbyte_cdk/sources/embedded/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
199
|
-
airbyte_cdk/sources/embedded/base_integration.py,sha256=0LbtEtWlnVdkYlweA5OJU4BIoyS6d4le5w9FsLn25Zc,2417
|
200
|
-
airbyte_cdk/sources/embedded/catalog.py,sha256=EAnLw9u5fXLNBLfWr_I0itA7OEHMWdqEaM_rWc_tCpI,1653
|
201
|
-
airbyte_cdk/sources/embedded/runner.py,sha256=S6cz7SWRTFfT_ZhP-zZYdiT9XRijulV_bMABal2h0OI,1493
|
202
|
-
airbyte_cdk/sources/embedded/tools.py,sha256=kp26Au9xH4hxsnDQoavxE_a418ZrRUAMBr2MeOU52VI,746
|
203
197
|
airbyte_cdk/sources/file_based/README.md,sha256=iMqww4VZ882jfNQIdljjDgqreKs-mkdtSrRKA94iX6A,11085
|
204
198
|
airbyte_cdk/sources/file_based/__init__.py,sha256=EaxHv_9ot-eRlUCR47ZMZ0IOtB-n0HH24om7Bfn-uuQ,868
|
205
199
|
airbyte_cdk/sources/file_based/availability_strategy/__init__.py,sha256=ddKQfUmk-Ls7LJaG8gtrqDybG3d8S7KXOAEjLeYLrTg,399
|
@@ -251,7 +245,7 @@ airbyte_cdk/sources/file_based/stream/identities_stream.py,sha256=DwgNU-jDp5vZ_W
|
|
251
245
|
airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py,sha256=i0Jn0zuAPomLa4pHSu9TQ3gAN5xXhNzPTYVwUDiDEyE,3523
|
252
246
|
airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
|
253
247
|
airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
|
254
|
-
airbyte_cdk/sources/http_logger.py,sha256=
|
248
|
+
airbyte_cdk/sources/http_logger.py,sha256=H93kPAujHhPmXNX0JSFG3D-SL6yEFA5PtKot9Hu3TYA,1690
|
255
249
|
airbyte_cdk/sources/message/__init__.py,sha256=y98fzHsQBwXwp2zEa4K5mxGFqjnx9lDn9O0pTk-VS4U,395
|
256
250
|
airbyte_cdk/sources/message/repository.py,sha256=SG7avgti_-dj8FcRHTTrhgLLGJbElv14_zIB0SH8AIc,4763
|
257
251
|
airbyte_cdk/sources/source.py,sha256=KIBBH5VLEb8BZ8B9aROlfaI6OLoJqKDPMJ10jkAR7nk,3611
|
@@ -285,15 +279,16 @@ airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py,sha256=
|
|
285
279
|
airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py,sha256=nbdkkHoN0NFeSs7YUFfzY1Lg5Jrt8fWY_ln3YrhY-Ko,544
|
286
280
|
airbyte_cdk/sources/streams/concurrent/partitions/types.py,sha256=frPVvHtY7vLxpGEbMQzNvF1Y52ZVyct9f1DDhGoRjwY,1166
|
287
281
|
airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
288
|
-
airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=
|
282
|
+
airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=JCRanZBAC8a0pPuzHCHiJ1irHgFkDd83l2K5jA1xRkU,6853
|
289
283
|
airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py,sha256=x8MLm1pTMfLNHvMF3P1ixYkYt_xjpbaIwnvhY_ofdBo,8076
|
284
|
+
airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py,sha256=bC6L82nsErXcFSPlxcdp4SneJ7qFuqCelP3-8svEh5E,3054
|
290
285
|
airbyte_cdk/sources/streams/core.py,sha256=jiYW6w8cjNjzXMd8U8Gt-02fYYU7b0ciXSSSnGvFRak,32219
|
291
286
|
airbyte_cdk/sources/streams/http/__init__.py,sha256=AGiEZ5B1Joi9ZnFpkJLT7F3QLpCAaBgAeVWy-1znmZw,311
|
292
287
|
airbyte_cdk/sources/streams/http/availability_strategy.py,sha256=sovoGFThZr-doMN9vJvTuJBrvkwQVIO0qTQO64pGZPY,2428
|
293
288
|
airbyte_cdk/sources/streams/http/error_handlers/__init__.py,sha256=nNO0bnD0ogDlR4AQrI-YmpcM911vWe83b7RJYgvjSYs,666
|
294
289
|
airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py,sha256=7fIkF00wD6bqIXtiG2QAgbje_xiQ5JTs99ibwR8LLXY,1030
|
295
290
|
airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py,sha256=1_1Gi2ImwTbh4SgIKCRh2P4kPaVeAeHc1ib6IrBfqKA,411
|
296
|
-
airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py,sha256=
|
291
|
+
airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py,sha256=drMniPygapMfOigh_ShJQvX6oqJaN4UA3cbRMkOgZJ4,3402
|
297
292
|
airbyte_cdk/sources/streams/http/error_handlers/error_handler.py,sha256=GuqP7U1eC9RPaiD4y4Mkf17vKcOXo2ENnMB-CUBLsbo,1164
|
298
293
|
airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py,sha256=xC93uB5BJd3iOnAXCrYLJTitWeGZlqzwe55VtsZqNnE,456
|
299
294
|
airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py,sha256=2gqececTxxUqO6aIkVNNXADg48Px5EHUwnXHL9KiPT8,4188
|
@@ -304,13 +299,13 @@ airbyte_cdk/sources/streams/http/http.py,sha256=0uariNq8OFnlX7iqOHwBhecxA-Hfd5hS
|
|
304
299
|
airbyte_cdk/sources/streams/http/http_client.py,sha256=tDE0ROtxjGMVphvsw8INvGMtZ97hIF-v47pZ3jIyiwc,23011
|
305
300
|
airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
|
306
301
|
airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
|
307
|
-
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=
|
302
|
+
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=P9U8vtcrZ3m0InSG2W0H4gTYTxjQxkIe6mhF9xvO8Ug,18824
|
308
303
|
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=Y3n7J-sk5yGjv_OxtY6Z6k0PEsFZmtIRi-x0KCbaHdA,1010
|
309
304
|
airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=C2j2uVfi9d-3KgHO3NGxIiFdfASjHOtsd6g_LWPYOAs,20311
|
310
305
|
airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
|
311
306
|
airbyte_cdk/sources/streams/permissions/identities_stream.py,sha256=9O9k6k18Xm3Zsiw_vnI_jsHXfMCQiek6V-jMkJJLxn8,2621
|
312
307
|
airbyte_cdk/sources/streams/utils/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
313
|
-
airbyte_cdk/sources/types.py,sha256=
|
308
|
+
airbyte_cdk/sources/types.py,sha256=x5Rkfpottg46qgr1-g_O4kN6v0Fd0sWHttdYsftyo7w,5148
|
314
309
|
airbyte_cdk/sources/utils/__init__.py,sha256=TTN6VUxVy6Is8BhYQZR5pxJGQh8yH4duXh4O1TiMiEY,118
|
315
310
|
airbyte_cdk/sources/utils/casing.py,sha256=QC-gV1O4e8DR4-bhdXieUPKm_JamzslVyfABLYYRSXA,256
|
316
311
|
airbyte_cdk/sources/utils/record_helper.py,sha256=jeB0mucudzna7Zvj-pCBbwFrbLJ36SlAWZTh5O4Fb9Y,2168
|
@@ -352,7 +347,7 @@ airbyte_cdk/utils/datetime_format_inferrer.py,sha256=Ne2cpk7Tx3eZDEW2Q3O7jnNOY9g
|
|
352
347
|
airbyte_cdk/utils/datetime_helpers.py,sha256=8mqzZ67Or2PBp7tLtrhh6XFv4wFzYsjCL_DOQJRaftI,17751
|
353
348
|
airbyte_cdk/utils/event_timing.py,sha256=aiuFmPU80buLlNdKq4fDTEqqhEIelHPF6AalFGwY8as,2557
|
354
349
|
airbyte_cdk/utils/is_cloud_environment.py,sha256=DayV32Irh-SdnJ0MnjvstwCJ66_l5oEsd8l85rZtHoc,574
|
355
|
-
airbyte_cdk/utils/mapping_helpers.py,sha256=
|
350
|
+
airbyte_cdk/utils/mapping_helpers.py,sha256=nWjOpnz_5QE9tY9-GtSWMPvYQL95kDD6k8KwwjUmrh0,6526
|
356
351
|
airbyte_cdk/utils/message_utils.py,sha256=OTzbkwN7AdMDA3iKYq1LKwfPFxpyEDfdgEF9BED3dkU,1366
|
357
352
|
airbyte_cdk/utils/oneof_option_config.py,sha256=N8EmWdYdwt0FM7fuShh6H8nj_r4KEL9tb2DJJtwsPow,1180
|
358
353
|
airbyte_cdk/utils/print_buffer.py,sha256=PhMOi0C4Z91kWKrSvCQXcp8qRh1uCimpIdvrg6voZIA,2810
|
@@ -361,9 +356,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
361
356
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
362
357
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
363
358
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
364
|
-
airbyte_cdk-6.37.
|
365
|
-
airbyte_cdk-6.37.
|
366
|
-
airbyte_cdk-6.37.
|
367
|
-
airbyte_cdk-6.37.
|
368
|
-
airbyte_cdk-6.37.
|
369
|
-
airbyte_cdk-6.37.
|
359
|
+
airbyte_cdk-6.37.2.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
360
|
+
airbyte_cdk-6.37.2.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
361
|
+
airbyte_cdk-6.37.2.dist-info/METADATA,sha256=FWmhFRUtj2Ro3GXW1iMMP-ujCoO5TAio7TKotvadR9A,6013
|
362
|
+
airbyte_cdk-6.37.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
363
|
+
airbyte_cdk-6.37.2.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
364
|
+
airbyte_cdk-6.37.2.dist-info/RECORD,,
|
@@ -1,136 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from dataclasses import dataclass
|
6
|
-
from typing import Any, Iterable, Mapping, Optional
|
7
|
-
|
8
|
-
from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
|
9
|
-
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
10
|
-
|
11
|
-
|
12
|
-
@dataclass
|
13
|
-
class GroupingPartitionRouter(PartitionRouter):
|
14
|
-
"""
|
15
|
-
A partition router that groups partitions from an underlying partition router into batches of a specified size.
|
16
|
-
This is useful for APIs that support filtering by multiple partition keys in a single request.
|
17
|
-
|
18
|
-
Attributes:
|
19
|
-
group_size (int): The number of partitions to include in each group.
|
20
|
-
underlying_partition_router (PartitionRouter): The partition router whose output will be grouped.
|
21
|
-
deduplicate (bool): If True, ensures unique partitions within each group by removing duplicates based on the partition key.
|
22
|
-
config (Config): The connector configuration.
|
23
|
-
parameters (Mapping[str, Any]): Additional parameters for interpolation and configuration.
|
24
|
-
"""
|
25
|
-
|
26
|
-
group_size: int
|
27
|
-
underlying_partition_router: PartitionRouter
|
28
|
-
config: Config
|
29
|
-
deduplicate: bool = True
|
30
|
-
|
31
|
-
def stream_slices(self) -> Iterable[StreamSlice]:
|
32
|
-
"""
|
33
|
-
Lazily groups partitions from the underlying partition router into batches of size `group_size`.
|
34
|
-
|
35
|
-
This method processes partitions one at a time from the underlying router, maintaining a batch buffer.
|
36
|
-
When the buffer reaches `group_size` or the underlying router is exhausted, it yields a grouped slice.
|
37
|
-
If deduplication is enabled, it tracks seen partition keys to ensure uniqueness within the current batch.
|
38
|
-
|
39
|
-
Yields:
|
40
|
-
Iterable[StreamSlice]: An iterable of StreamSlice objects, where each slice contains a batch of partition values.
|
41
|
-
"""
|
42
|
-
batch = []
|
43
|
-
seen_keys = set()
|
44
|
-
|
45
|
-
# Iterate over partitions lazily from the underlying router
|
46
|
-
for partition in self.underlying_partition_router.stream_slices():
|
47
|
-
# Extract the partition key (assuming single key-value pair, e.g., {"board_ids": value})
|
48
|
-
key = next(iter(partition.partition.values()), None)
|
49
|
-
|
50
|
-
# Skip duplicates if deduplication is enabled
|
51
|
-
if self.deduplicate and key in seen_keys:
|
52
|
-
continue
|
53
|
-
|
54
|
-
# Add partition to the batch
|
55
|
-
batch.append(partition)
|
56
|
-
if self.deduplicate:
|
57
|
-
seen_keys.add(key)
|
58
|
-
|
59
|
-
# Yield the batch when it reaches the group_size
|
60
|
-
if len(batch) == self.group_size:
|
61
|
-
yield self._create_grouped_slice(batch)
|
62
|
-
batch = [] # Reset the batch
|
63
|
-
|
64
|
-
# Yield any remaining partitions if the batch isn't empty
|
65
|
-
if batch:
|
66
|
-
yield self._create_grouped_slice(batch)
|
67
|
-
|
68
|
-
def _create_grouped_slice(self, batch: list[StreamSlice]) -> StreamSlice:
|
69
|
-
"""
|
70
|
-
Creates a grouped StreamSlice from a batch of partitions, aggregating extra fields into a dictionary with list values.
|
71
|
-
|
72
|
-
Args:
|
73
|
-
batch (list[StreamSlice]): A list of StreamSlice objects to group.
|
74
|
-
|
75
|
-
Returns:
|
76
|
-
StreamSlice: A single StreamSlice with combined partition and extra field values.
|
77
|
-
"""
|
78
|
-
# Combine partition values into a single dict with lists
|
79
|
-
grouped_partition = {
|
80
|
-
key: [p.partition.get(key) for p in batch] for key in batch[0].partition.keys()
|
81
|
-
}
|
82
|
-
|
83
|
-
# Aggregate extra fields into a dict with list values
|
84
|
-
extra_fields_dict = (
|
85
|
-
{
|
86
|
-
key: [p.extra_fields.get(key) for p in batch]
|
87
|
-
for key in set().union(*(p.extra_fields.keys() for p in batch if p.extra_fields))
|
88
|
-
}
|
89
|
-
if any(p.extra_fields for p in batch)
|
90
|
-
else {}
|
91
|
-
)
|
92
|
-
return StreamSlice(
|
93
|
-
partition=grouped_partition,
|
94
|
-
cursor_slice={}, # Cursor is managed by the underlying router or incremental sync
|
95
|
-
extra_fields=extra_fields_dict,
|
96
|
-
)
|
97
|
-
|
98
|
-
def get_request_params(
|
99
|
-
self,
|
100
|
-
stream_state: Optional[StreamState] = None,
|
101
|
-
stream_slice: Optional[StreamSlice] = None,
|
102
|
-
next_page_token: Optional[Mapping[str, Any]] = None,
|
103
|
-
) -> Mapping[str, Any]:
|
104
|
-
return {}
|
105
|
-
|
106
|
-
def get_request_headers(
|
107
|
-
self,
|
108
|
-
stream_state: Optional[StreamState] = None,
|
109
|
-
stream_slice: Optional[StreamSlice] = None,
|
110
|
-
next_page_token: Optional[Mapping[str, Any]] = None,
|
111
|
-
) -> Mapping[str, Any]:
|
112
|
-
return {}
|
113
|
-
|
114
|
-
def get_request_body_data(
|
115
|
-
self,
|
116
|
-
stream_state: Optional[StreamState] = None,
|
117
|
-
stream_slice: Optional[StreamSlice] = None,
|
118
|
-
next_page_token: Optional[Mapping[str, Any]] = None,
|
119
|
-
) -> Mapping[str, Any]:
|
120
|
-
return {}
|
121
|
-
|
122
|
-
def get_request_body_json(
|
123
|
-
self,
|
124
|
-
stream_state: Optional[StreamState] = None,
|
125
|
-
stream_slice: Optional[StreamSlice] = None,
|
126
|
-
next_page_token: Optional[Mapping[str, Any]] = None,
|
127
|
-
) -> Mapping[str, Any]:
|
128
|
-
return {}
|
129
|
-
|
130
|
-
def set_initial_state(self, stream_state: StreamState) -> None:
|
131
|
-
"""Delegate state initialization to the underlying partition router."""
|
132
|
-
self.underlying_partition_router.set_initial_state(stream_state)
|
133
|
-
|
134
|
-
def get_stream_state(self) -> Optional[Mapping[str, StreamState]]:
|
135
|
-
"""Delegate state retrieval to the underlying partition router."""
|
136
|
-
return self.underlying_partition_router.get_stream_state()
|
@@ -1,61 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from abc import ABC, abstractmethod
|
6
|
-
from typing import Generic, Iterable, Optional, TypeVar
|
7
|
-
|
8
|
-
from airbyte_cdk.connector import TConfig
|
9
|
-
from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type
|
10
|
-
from airbyte_cdk.sources.embedded.catalog import (
|
11
|
-
create_configured_catalog,
|
12
|
-
get_stream,
|
13
|
-
get_stream_names,
|
14
|
-
)
|
15
|
-
from airbyte_cdk.sources.embedded.runner import SourceRunner
|
16
|
-
from airbyte_cdk.sources.embedded.tools import get_defined_id
|
17
|
-
from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
|
18
|
-
|
19
|
-
TOutput = TypeVar("TOutput")
|
20
|
-
|
21
|
-
|
22
|
-
class BaseEmbeddedIntegration(ABC, Generic[TConfig, TOutput]):
|
23
|
-
def __init__(self, runner: SourceRunner[TConfig], config: TConfig):
|
24
|
-
check_config_against_spec_or_exit(config, runner.spec())
|
25
|
-
|
26
|
-
self.source = runner
|
27
|
-
self.config = config
|
28
|
-
|
29
|
-
self.last_state: Optional[AirbyteStateMessage] = None
|
30
|
-
|
31
|
-
@abstractmethod
|
32
|
-
def _handle_record(self, record: AirbyteRecordMessage, id: Optional[str]) -> Optional[TOutput]:
|
33
|
-
"""
|
34
|
-
Turn an Airbyte record into the appropriate output type for the integration.
|
35
|
-
"""
|
36
|
-
pass
|
37
|
-
|
38
|
-
def _load_data(
|
39
|
-
self, stream_name: str, state: Optional[AirbyteStateMessage] = None
|
40
|
-
) -> Iterable[TOutput]:
|
41
|
-
catalog = self.source.discover(self.config)
|
42
|
-
stream = get_stream(catalog, stream_name)
|
43
|
-
if not stream:
|
44
|
-
raise ValueError(
|
45
|
-
f"Stream {stream_name} not found, the following streams are available: {', '.join(get_stream_names(catalog))}"
|
46
|
-
)
|
47
|
-
if SyncMode.incremental not in stream.supported_sync_modes:
|
48
|
-
configured_catalog = create_configured_catalog(stream, sync_mode=SyncMode.full_refresh)
|
49
|
-
else:
|
50
|
-
configured_catalog = create_configured_catalog(stream, sync_mode=SyncMode.incremental)
|
51
|
-
|
52
|
-
for message in self.source.read(self.config, configured_catalog, state):
|
53
|
-
if message.type == Type.RECORD:
|
54
|
-
output = self._handle_record(
|
55
|
-
message.record,
|
56
|
-
get_defined_id(stream, message.record.data), # type: ignore[union-attr, arg-type]
|
57
|
-
)
|
58
|
-
if output:
|
59
|
-
yield output
|
60
|
-
elif message.type is Type.STATE and message.state:
|
61
|
-
self.last_state = message.state
|
@@ -1,57 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from typing import List, Optional
|
6
|
-
|
7
|
-
from airbyte_cdk.models import (
|
8
|
-
AirbyteCatalog,
|
9
|
-
AirbyteStream,
|
10
|
-
ConfiguredAirbyteCatalog,
|
11
|
-
ConfiguredAirbyteStream,
|
12
|
-
DestinationSyncMode,
|
13
|
-
SyncMode,
|
14
|
-
)
|
15
|
-
from airbyte_cdk.sources.embedded.tools import get_first
|
16
|
-
|
17
|
-
|
18
|
-
def get_stream(catalog: AirbyteCatalog, stream_name: str) -> Optional[AirbyteStream]:
|
19
|
-
return get_first(catalog.streams, lambda s: s.name == stream_name)
|
20
|
-
|
21
|
-
|
22
|
-
def get_stream_names(catalog: AirbyteCatalog) -> List[str]:
|
23
|
-
return [stream.name for stream in catalog.streams]
|
24
|
-
|
25
|
-
|
26
|
-
def to_configured_stream(
|
27
|
-
stream: AirbyteStream,
|
28
|
-
sync_mode: SyncMode = SyncMode.full_refresh,
|
29
|
-
destination_sync_mode: DestinationSyncMode = DestinationSyncMode.append,
|
30
|
-
cursor_field: Optional[List[str]] = None,
|
31
|
-
primary_key: Optional[List[List[str]]] = None,
|
32
|
-
) -> ConfiguredAirbyteStream:
|
33
|
-
return ConfiguredAirbyteStream(
|
34
|
-
stream=stream,
|
35
|
-
sync_mode=sync_mode,
|
36
|
-
destination_sync_mode=destination_sync_mode,
|
37
|
-
cursor_field=cursor_field,
|
38
|
-
primary_key=primary_key,
|
39
|
-
)
|
40
|
-
|
41
|
-
|
42
|
-
def to_configured_catalog(
|
43
|
-
configured_streams: List[ConfiguredAirbyteStream],
|
44
|
-
) -> ConfiguredAirbyteCatalog:
|
45
|
-
return ConfiguredAirbyteCatalog(streams=configured_streams)
|
46
|
-
|
47
|
-
|
48
|
-
def create_configured_catalog(
|
49
|
-
stream: AirbyteStream, sync_mode: SyncMode = SyncMode.full_refresh
|
50
|
-
) -> ConfiguredAirbyteCatalog:
|
51
|
-
configured_streams = [
|
52
|
-
to_configured_stream(
|
53
|
-
stream, sync_mode=sync_mode, primary_key=stream.source_defined_primary_key
|
54
|
-
)
|
55
|
-
]
|
56
|
-
|
57
|
-
return to_configured_catalog(configured_streams)
|
@@ -1,57 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
|
6
|
-
import logging
|
7
|
-
from abc import ABC, abstractmethod
|
8
|
-
from typing import Generic, Iterable, Optional
|
9
|
-
|
10
|
-
from airbyte_cdk.connector import TConfig
|
11
|
-
from airbyte_cdk.models import (
|
12
|
-
AirbyteCatalog,
|
13
|
-
AirbyteMessage,
|
14
|
-
AirbyteStateMessage,
|
15
|
-
ConfiguredAirbyteCatalog,
|
16
|
-
ConnectorSpecification,
|
17
|
-
)
|
18
|
-
from airbyte_cdk.sources.source import Source
|
19
|
-
|
20
|
-
|
21
|
-
class SourceRunner(ABC, Generic[TConfig]):
|
22
|
-
@abstractmethod
|
23
|
-
def spec(self) -> ConnectorSpecification:
|
24
|
-
pass
|
25
|
-
|
26
|
-
@abstractmethod
|
27
|
-
def discover(self, config: TConfig) -> AirbyteCatalog:
|
28
|
-
pass
|
29
|
-
|
30
|
-
@abstractmethod
|
31
|
-
def read(
|
32
|
-
self,
|
33
|
-
config: TConfig,
|
34
|
-
catalog: ConfiguredAirbyteCatalog,
|
35
|
-
state: Optional[AirbyteStateMessage],
|
36
|
-
) -> Iterable[AirbyteMessage]:
|
37
|
-
pass
|
38
|
-
|
39
|
-
|
40
|
-
class CDKRunner(SourceRunner[TConfig]):
|
41
|
-
def __init__(self, source: Source, name: str):
|
42
|
-
self._source = source
|
43
|
-
self._logger = logging.getLogger(name)
|
44
|
-
|
45
|
-
def spec(self) -> ConnectorSpecification:
|
46
|
-
return self._source.spec(self._logger)
|
47
|
-
|
48
|
-
def discover(self, config: TConfig) -> AirbyteCatalog:
|
49
|
-
return self._source.discover(self._logger, config)
|
50
|
-
|
51
|
-
def read(
|
52
|
-
self,
|
53
|
-
config: TConfig,
|
54
|
-
catalog: ConfiguredAirbyteCatalog,
|
55
|
-
state: Optional[AirbyteStateMessage],
|
56
|
-
) -> Iterable[AirbyteMessage]:
|
57
|
-
return self._source.read(self._logger, config, catalog, state=[state] if state else [])
|
@@ -1,27 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from typing import Any, Callable, Dict, Iterable, Optional
|
6
|
-
|
7
|
-
import dpath
|
8
|
-
|
9
|
-
from airbyte_cdk.models import AirbyteStream
|
10
|
-
|
11
|
-
|
12
|
-
def get_first(
|
13
|
-
iterable: Iterable[Any], predicate: Callable[[Any], bool] = lambda m: True
|
14
|
-
) -> Optional[Any]:
|
15
|
-
return next(filter(predicate, iterable), None)
|
16
|
-
|
17
|
-
|
18
|
-
def get_defined_id(stream: AirbyteStream, data: Dict[str, Any]) -> Optional[str]:
|
19
|
-
if not stream.source_defined_primary_key:
|
20
|
-
return None
|
21
|
-
primary_key = []
|
22
|
-
for key in stream.source_defined_primary_key:
|
23
|
-
try:
|
24
|
-
primary_key.append(str(dpath.get(data, key)))
|
25
|
-
except KeyError:
|
26
|
-
primary_key.append("__not_found__")
|
27
|
-
return "_".join(primary_key)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|