airbyte-cdk 6.37.0.dev1__py3-none-any.whl → 6.37.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. airbyte_cdk/connector_builder/models.py +16 -14
  2. airbyte_cdk/connector_builder/test_reader/helpers.py +120 -22
  3. airbyte_cdk/connector_builder/test_reader/message_grouper.py +16 -3
  4. airbyte_cdk/connector_builder/test_reader/types.py +9 -1
  5. airbyte_cdk/sources/declarative/auth/token_provider.py +1 -0
  6. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +43 -7
  7. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +7 -1
  8. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +77 -48
  9. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +13 -2
  10. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -0
  11. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +83 -17
  12. airbyte_cdk/sources/declarative/interpolation/macros.py +2 -0
  13. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +37 -50
  14. airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +18 -4
  15. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +171 -70
  16. airbyte_cdk/sources/declarative/partition_routers/__init__.py +0 -4
  17. airbyte_cdk/sources/declarative/requesters/README.md +5 -5
  18. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +60 -17
  19. airbyte_cdk/sources/declarative/requesters/http_requester.py +49 -17
  20. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +25 -4
  21. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +6 -1
  22. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +7 -2
  23. airbyte_cdk/sources/declarative/requesters/requester.py +7 -1
  24. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +10 -3
  25. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +21 -4
  26. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +2 -2
  27. airbyte_cdk/sources/http_logger.py +3 -0
  28. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +2 -1
  29. airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
  30. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +3 -3
  31. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -0
  32. airbyte_cdk/sources/types.py +1 -0
  33. airbyte_cdk/utils/mapping_helpers.py +18 -1
  34. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/METADATA +4 -4
  35. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/RECORD +39 -44
  36. airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py +0 -136
  37. airbyte_cdk/sources/embedded/__init__.py +0 -3
  38. airbyte_cdk/sources/embedded/base_integration.py +0 -61
  39. airbyte_cdk/sources/embedded/catalog.py +0 -57
  40. airbyte_cdk/sources/embedded/runner.py +0 -57
  41. airbyte_cdk/sources/embedded/tools.py +0 -27
  42. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/LICENSE.txt +0 -0
  43. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/LICENSE_SHORT +0 -0
  44. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/WHEEL +0 -0
  45. {airbyte_cdk-6.37.0.dev1.dist-info → airbyte_cdk-6.37.2.dist-info}/entry_points.txt +0 -0
@@ -9,12 +9,12 @@ airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz
9
9
  airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
10
10
  airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=BntqkP63RBPvGCtB3CrLLtYplfSlBR42kwXyyk4YGas,4268
11
11
  airbyte_cdk/connector_builder/main.py,sha256=ubAPE0Oo5gjZOa-KMtLLJQkc8_inUpFR3sIb2DEh2No,3722
12
- airbyte_cdk/connector_builder/models.py,sha256=uCHpOdJx2PyZtIqk-mt9eSVuFMQoEqrW-9sjCz0Z-AQ,1500
12
+ airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
13
13
  airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
14
- airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=niu_EhzwVXnvtzj2Rf_unrxqLRC4Twbe-t17HyNoRJY,23662
15
- airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=L0xKpWQhfRecKblI3uYO9twPTPJYJzlOhK2P8zHWXRU,6487
14
+ airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=Iczn-_iczS2CaIAunWwyFcX0uLTra8Wh9JVfzm1Gfxo,26765
15
+ airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=84BAEPIBHMq3WCfO14WNvh_q7OsjGgDt0q1FTu8eW-w,6918
16
16
  airbyte_cdk/connector_builder/test_reader/reader.py,sha256=GurMB4ITO_PntvhIHSJkXbhynLilI4DObY5A2axavXo,20667
17
- airbyte_cdk/connector_builder/test_reader/types.py,sha256=jP28aOlCS8Q6V7jMksfJKsuAJ-m2dNYiXaUjYvb0DBA,2404
17
+ airbyte_cdk/connector_builder/test_reader/types.py,sha256=hPZG3jO03kBaPyW94NI3JHRS1jxXGSNBcN1HFzOxo5Y,2528
18
18
  airbyte_cdk/destinations/__init__.py,sha256=FyDp28PT_YceJD5HDFhA-mrGfX9AONIyMQ4d68CHNxQ,213
19
19
  airbyte_cdk/destinations/destination.py,sha256=CIq-yb8C_0QvcKCtmStaHfiqn53GEfRAIGGCkJhKP1Q,5880
20
20
  airbyte_cdk/destinations/vector_db_based/README.md,sha256=QAe8c_1Afme4r2TCE10cTSaxUE3zgCBuArSuRQqK8tA,2115
@@ -60,22 +60,22 @@ airbyte_cdk/sources/declarative/auth/jwt.py,sha256=SICqNsN2Cn_EgKadIgWuZpQxuMHyz
60
60
  airbyte_cdk/sources/declarative/auth/oauth.py,sha256=SUfib1oSzlyRRnOSg8Bui73mfyrcyr9OssdchbKdu4s,14162
61
61
  airbyte_cdk/sources/declarative/auth/selective_authenticator.py,sha256=qGwC6YsCldr1bIeKG6Qo-A9a5cTdHw-vcOn3OtQrS4c,1540
62
62
  airbyte_cdk/sources/declarative/auth/token.py,sha256=2EnE78EhBOY9hbeZnQJ9AuFaM-G7dccU-oKo_LThRQk,11070
63
- airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=9CuSsmOoHkvlc4k-oZ3Jx5luAgfTMm1I_5HOZxw7wMU,3075
63
+ airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=Jzuxlmt1_-_aFC_n0OmP8L1nDOacLzbEVVx3kjdX_W8,3104
64
64
  airbyte_cdk/sources/declarative/checks/__init__.py,sha256=nsVV5Bo0E_tBNd8A4Xdsdb-75PpcLo5RQu2RQ_Gv-ME,806
65
65
  airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py,sha256=HUktywjI8pqOeED08UGqponUSwxs2TOAECTowlWlrRE,2138
66
66
  airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQrilWCfJmncBzXCZ18ptRNip3XA,2139
67
67
  airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
68
68
  airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
69
69
  airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
70
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=MRnIdGeKPk1dO9-4eWRHa7mI6Ay_7szGo9H1RJSZDb8,24453
70
+ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=rAp-sgld4n8Tmybz-51m7VcYXqKwzKDpCJVr1elmkRc,26824
71
71
  airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
72
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
72
+ airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=0qs4hhmh_XOy2B4MHCn2qVMM79C6MizIBqnvpZj1aSE,2923
73
73
  airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
74
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=d8jbuP86iBhEhtbtw5dGg9b8U93KkDACgAO4EYW2lzg,146344
74
+ airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=n8hJVquDj00_VS_I0B2QgwYNcNcfsVZdkajAKArcOHU,147487
75
75
  airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
76
76
  airbyte_cdk/sources/declarative/declarative_stream.py,sha256=venZjfpvtqr3oFSuvMBWtn4h9ayLhD4L65ACuXCDZ64,10445
77
77
  airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
78
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=kg5_kNlhXj8p9GZFztlbG16pk1XcMtP9ezqulq1VNg4,4545
78
+ airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=DJbWaaJ5LHCBpyWz-4bEw8rqtJYqabEYZtxnfRtWFE0,4946
79
79
  airbyte_cdk/sources/declarative/decoders/decoder.py,sha256=sl-Gt8lXi7yD2Q-sD8je5QS2PbgrgsYjxRLWsay7DMc,826
80
80
  airbyte_cdk/sources/declarative/decoders/json_decoder.py,sha256=BdWpXXPhEGf_zknggJmhojLosmxuw51RBVTS0jvdCPc,2080
81
81
  airbyte_cdk/sources/declarative/decoders/noop_decoder.py,sha256=iZh0yKY_JzgBnJWiubEusf5c0o6Khd-8EWFWT-8EgFo,542
@@ -89,10 +89,10 @@ airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8V
89
89
  airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
90
90
  airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
91
91
  airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=HCqx7IyENM_aRF4it2zJN26_vDu6WeP8XgCxQWHUvcY,6934
92
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=LhqGDfX06_dDYLKsIVnwQ_nAWCln-v8PV7Wgt_QVeTI,6533
92
+ airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=L6hQV7bdwEp8y-TBMeQY-xmrNyRggL14lKXdWnzYFfA,6622
93
93
  airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
94
94
  airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
95
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=Pg2phEFT9T8AzUjK6hVhn0rgR3yY6JPF-Dfv0g1m5dQ,19191
95
+ airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=MT5JbdEbnPzk3VWZGGvThe4opoX5dHhSXFrnTRYC6dg,22210
96
96
  airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
97
97
  airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
98
98
  airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
@@ -107,28 +107,27 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha
107
107
  airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkHqGlfa87G6VYMtBAQWin7ECKpfMdrDcg0JO5_rhc,3212
108
108
  airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
109
109
  airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
110
- airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=ajgVJT9sZBUFZUDLjzyPWupCNXt1HvzbCq-4yv9aY3c,5042
110
+ airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=uuXBZUWDWM-sPcUKjNSPRN657QhNQCx_hnhTuJj2zOA,5129
111
111
  airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=TN6GCgLXaWDONTaJwQ3A5ELqC-sxwKz-UYSraJYB-dI,17078
112
112
  airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
113
113
  airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
114
114
  airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
115
115
  airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
116
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=Uk8sJRXs763ym_UCBzW2YGDzLPwTvzMQt_J0a3M-5zA,103309
116
+ airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=DfbPi512ovaBSWDICJfjIkC3pXDn2aNr1BP-eiLOLyA,103556
117
117
  airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
118
- airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=958MMX6_ZOJUlDDdNr9Krosgi2bCKGx2Z765M2Woz18,5505
118
+ airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=jDw_TttD3_hpfevXOH-0Ws0eRuqt6wvED0BqosGPRjI,5938
119
119
  airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
120
120
  airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=CXwTfD3wSQq3okcqwigpprbHhSURUokh4GK2OmOyKC8,9132
121
121
  airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
122
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=nY_s3xidDFJTuxlxbLVfjlZIxMhaXtwAg7tnxdkgdXg,135237
123
- airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
122
+ airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Mx0KJGbqIZeUWduKy-UvpVH-DRm0pzXDcz203r69oNY,140619
123
+ airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=HJ-Syp3p7RpyR_OK0X_a2kSyISfu3W-PKrRI16iY0a8,957
124
124
  airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
125
125
  airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
126
- airbyte_cdk/sources/declarative/partition_routers/grouping_partition_router.py,sha256=PFoY931wC1i7Elphrd7LCFUPYKOTPEovLXC-mvkQow0,5531
127
126
  airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=tmGGpMoOBmaMfhVZq53AEWxoHm2lmNVi6hA2_IVEnAA,4882
128
127
  airbyte_cdk/sources/declarative/partition_routers/partition_router.py,sha256=YyEIzdmLd1FjbVP3QbQ2VFCLW_P-OGbVh6VpZShp54k,2218
129
128
  airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py,sha256=SKzKjSyfccq4dxGIh-J6ejrgkCHzaiTIazmbmeQiRD4,1942
130
129
  airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py,sha256=LlWj-Ofs-xfjlqmDzH8OYpyblP2Pb8bPDdR9g1UZyt0,17693
131
- airbyte_cdk/sources/declarative/requesters/README.md,sha256=eL1I4iLkxaw7hJi9S9d18_XcRl-R8lUSjqBVJJzvXmg,2656
130
+ airbyte_cdk/sources/declarative/requesters/README.md,sha256=DQll2qsIzzTiiP35kJp16ONpr7cFeUQNgPfhl5krB24,2675
132
131
  airbyte_cdk/sources/declarative/requesters/__init__.py,sha256=d7a3OoHbqaJDyyPli3nqqJ2yAW_SLX6XDaBAKOwvpxw,364
133
132
  airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py,sha256=SkEDcJxlT1683rNx93K9whoS0OyUukkuOfToGtgpF58,776
134
133
  airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py,sha256=1WZdpFmWL6W_Dko0qjflTaKIWeqt8jHT-D6HcujIp3s,884
@@ -143,12 +142,12 @@ airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.
143
142
  airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py,sha256=q0YkeYUUWO6iErUy0vjqiOkhg8_9d5YcCmtlpXAJJ9E,1314
144
143
  airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py,sha256=Tan66odx8VHzfdyyXMQkXz2pJYksllGqvxmpoajgcK4,669
145
144
  airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py,sha256=E-fQbt4ShfxZVoqfnmOx69C6FUPWZz8BIqI3DN9Kcjs,7935
146
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=3GtOefPH08evlSUxaILkiKLTHbIspFY4qd5B3ZqNE60,10063
147
- airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=pR2uR5b9eGyvYIOYwus3mz3OaqRu1ozwja_ys1SE7hc,14952
145
+ airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=4wpP0ZNTMLugi-Rc1OFdFaxWfRZSl45nzhHqMFCE8SQ,11924
146
+ airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=KS2nEUKMnyow8HvvfJJbWdPjlglIWSOZIAxrwkmXCI4,16356
148
147
  airbyte_cdk/sources/declarative/requesters/paginators/__init__.py,sha256=uArbKs9JKNCt7t9tZoeWwjDpyI1HoPp29FNW0JzvaEM,644
149
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=ZW4lwWNAzb4zL0jKc-HjowP5-y0Zg9xi0YlK6tkx_XY,12057
150
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=j6j9QRPaTbKQ2N661RFVKthhkWiodEp6ut0tKeEd0Ng,2019
151
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py,sha256=OlN-y0PEOMzlUNUh3pzonoTpIJpGwkP4ibFengvpLVU,2230
148
+ airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=SB-Af3CRb4mJwhm4EKNxzl_PK2w5QS4tqrSNNMO2IV4,12760
149
+ airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py,sha256=b1-zKxYOUMHn7ahdWpzKEzfG4A7s_WQWy-vzRqZWzME,2152
150
+ airbyte_cdk/sources/declarative/requesters/paginators/paginator.py,sha256=TzJF1Q-CFlsHF9lMSfmnGCxRYm9_UQCmBcHYQpc7F30,2376
152
151
  airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py,sha256=2gly8fuZpDNwtu1Qg6oE2jBLGqQRdzSLJdnpk_iDV6I,767
153
152
  airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py,sha256=yLzzK5YIRTkXd2Z-BS__AZXuTd6HXjJIxq05K-lQoxI,3898
154
153
  airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py,sha256=WvGt_DTFcAgTR-NHrlrR7B71yG-L6jmfW-Gwm9iYzjY,3624
@@ -164,15 +163,15 @@ airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_
164
163
  airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py,sha256=vOsdHfWHiTFc89WENHPv1hcxLgdzycMXVT_IEtLuhfs,5012
165
164
  airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py,sha256=8YRiDzjYvqJ-aMmKFcjqzv_-e8OZ5QG_TbpZ-nuCu6s,2590
166
165
  airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
167
- airbyte_cdk/sources/declarative/requesters/requester.py,sha256=iVVpXQ4KEd9OyZNwmOofMvx7_06i8ZRxGo3aNTrEQLM,4946
166
+ airbyte_cdk/sources/declarative/requesters/requester.py,sha256=OcDzuCBgD1owK_lBPG0KbRRHRn9kzbuRveU4HejDiv4,5116
168
167
  airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=NiDcz5qi8HPsfX94MUmnX0Rgs_kQXGvucOmJjNWlxKQ,1207
169
168
  airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb9artL4ZkeqN8RmEykHH6FJgqXD7fCEnh1X0,1936
170
169
  airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
171
170
  airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
172
171
  airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=ix9m1dkR69DcXCXUKC5RK_ZZM7ojTLBQ4IkWQTfmfCk,456
173
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=2oQn_vo7uJKp4pdMnsF5CG5Iwc9rkPeEOLoAm_9bcus,3222
172
+ airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=dwYZ70eg9DKHEqZydHhMFPkEILbNcXu7E-djOCikNgI,3530
174
173
  airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
175
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=bOAKQLgMv1Vca-ozMPRVAg1V5nkyUoPwqC02lKpnLiM,24575
174
+ airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=fDhc6dMx75UImh1_TfLm4Le59tsHpqIUZnau7uIJyYw,25043
176
175
  airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
177
176
  airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=KTACrIE23a83wsm3Rd9Eb4K6-20lrGqYxTHNp9yxsso,1820
178
177
  airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=J8Q_iJYhcSQLWyt0bTZCbDAGpxt9G8FCc6Q9jtGsNzw,10703
@@ -190,16 +189,11 @@ airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py,sha256=1
190
189
  airbyte_cdk/sources/declarative/transformations/flatten_fields.py,sha256=yT3owG6rMKaRX-LJ_T-jSTnh1B5NoAHyH4YZN9yOvE8,1758
191
190
  airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py,sha256=vbIn6ump-Ut6g20yMub7PFoPBhOKVtrHSAUdcOUdLfw,1999
192
191
  airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py,sha256=RTs5KX4V3hM7A6QN1WlGF21YccTIyNH6qQI9IMb__hw,670
193
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py,sha256=43zwe6_F5ba5C4eY0RgXxPz7ndPKZfXGChHepFn-2lk,2263
192
+ airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py,sha256=_3ldEbsA7tQK-zzeU_cG86D1_1SY3wAo1vHE0zXrOck,2265
194
193
  airbyte_cdk/sources/declarative/transformations/remove_fields.py,sha256=EwUP0SZ2p4GRJ6Q8CUzlz9dcUeEidEFDlI2IBye2tlc,2745
195
194
  airbyte_cdk/sources/declarative/transformations/transformation.py,sha256=4sXtx9cNY2EHUPq-xHvDs8GQEBUy3Eo6TkRLKHPXx68,1161
196
195
  airbyte_cdk/sources/declarative/types.py,sha256=yqx0xlZv_76tkC7fqJKefmvl4GJJ8mXbeddwVV8XRJU,778
197
196
  airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256=MsKSAqtpwIqJfYOiUX01RbqMeTy7pvBoguvyTWrL7pI,2390
198
- airbyte_cdk/sources/embedded/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
199
- airbyte_cdk/sources/embedded/base_integration.py,sha256=0LbtEtWlnVdkYlweA5OJU4BIoyS6d4le5w9FsLn25Zc,2417
200
- airbyte_cdk/sources/embedded/catalog.py,sha256=EAnLw9u5fXLNBLfWr_I0itA7OEHMWdqEaM_rWc_tCpI,1653
201
- airbyte_cdk/sources/embedded/runner.py,sha256=S6cz7SWRTFfT_ZhP-zZYdiT9XRijulV_bMABal2h0OI,1493
202
- airbyte_cdk/sources/embedded/tools.py,sha256=kp26Au9xH4hxsnDQoavxE_a418ZrRUAMBr2MeOU52VI,746
203
197
  airbyte_cdk/sources/file_based/README.md,sha256=iMqww4VZ882jfNQIdljjDgqreKs-mkdtSrRKA94iX6A,11085
204
198
  airbyte_cdk/sources/file_based/__init__.py,sha256=EaxHv_9ot-eRlUCR47ZMZ0IOtB-n0HH24om7Bfn-uuQ,868
205
199
  airbyte_cdk/sources/file_based/availability_strategy/__init__.py,sha256=ddKQfUmk-Ls7LJaG8gtrqDybG3d8S7KXOAEjLeYLrTg,399
@@ -251,7 +245,7 @@ airbyte_cdk/sources/file_based/stream/identities_stream.py,sha256=DwgNU-jDp5vZ_W
251
245
  airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py,sha256=i0Jn0zuAPomLa4pHSu9TQ3gAN5xXhNzPTYVwUDiDEyE,3523
252
246
  airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
253
247
  airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
254
- airbyte_cdk/sources/http_logger.py,sha256=l_1fk5YwdonZ1wvAsTwjj6d36fj2WrVraIAMj5jTQdM,1575
248
+ airbyte_cdk/sources/http_logger.py,sha256=H93kPAujHhPmXNX0JSFG3D-SL6yEFA5PtKot9Hu3TYA,1690
255
249
  airbyte_cdk/sources/message/__init__.py,sha256=y98fzHsQBwXwp2zEa4K5mxGFqjnx9lDn9O0pTk-VS4U,395
256
250
  airbyte_cdk/sources/message/repository.py,sha256=SG7avgti_-dj8FcRHTTrhgLLGJbElv14_zIB0SH8AIc,4763
257
251
  airbyte_cdk/sources/source.py,sha256=KIBBH5VLEb8BZ8B9aROlfaI6OLoJqKDPMJ10jkAR7nk,3611
@@ -285,15 +279,16 @@ airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py,sha256=
285
279
  airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py,sha256=nbdkkHoN0NFeSs7YUFfzY1Lg5Jrt8fWY_ln3YrhY-Ko,544
286
280
  airbyte_cdk/sources/streams/concurrent/partitions/types.py,sha256=frPVvHtY7vLxpGEbMQzNvF1Y52ZVyct9f1DDhGoRjwY,1166
287
281
  airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
288
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=CXHUMOhndu-LOKgsnNTItv5s5qrKpmJDeHOzlH1nBy8,6819
282
+ airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=JCRanZBAC8a0pPuzHCHiJ1irHgFkDd83l2K5jA1xRkU,6853
289
283
  airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py,sha256=x8MLm1pTMfLNHvMF3P1ixYkYt_xjpbaIwnvhY_ofdBo,8076
284
+ airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py,sha256=bC6L82nsErXcFSPlxcdp4SneJ7qFuqCelP3-8svEh5E,3054
290
285
  airbyte_cdk/sources/streams/core.py,sha256=jiYW6w8cjNjzXMd8U8Gt-02fYYU7b0ciXSSSnGvFRak,32219
291
286
  airbyte_cdk/sources/streams/http/__init__.py,sha256=AGiEZ5B1Joi9ZnFpkJLT7F3QLpCAaBgAeVWy-1znmZw,311
292
287
  airbyte_cdk/sources/streams/http/availability_strategy.py,sha256=sovoGFThZr-doMN9vJvTuJBrvkwQVIO0qTQO64pGZPY,2428
293
288
  airbyte_cdk/sources/streams/http/error_handlers/__init__.py,sha256=nNO0bnD0ogDlR4AQrI-YmpcM911vWe83b7RJYgvjSYs,666
294
289
  airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py,sha256=7fIkF00wD6bqIXtiG2QAgbje_xiQ5JTs99ibwR8LLXY,1030
295
290
  airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py,sha256=1_1Gi2ImwTbh4SgIKCRh2P4kPaVeAeHc1ib6IrBfqKA,411
296
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py,sha256=Z0gHhbBSFNWuatzCtBOQt9C-BkiN1G76wr6yl0qodEU,3376
291
+ airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py,sha256=drMniPygapMfOigh_ShJQvX6oqJaN4UA3cbRMkOgZJ4,3402
297
292
  airbyte_cdk/sources/streams/http/error_handlers/error_handler.py,sha256=GuqP7U1eC9RPaiD4y4Mkf17vKcOXo2ENnMB-CUBLsbo,1164
298
293
  airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py,sha256=xC93uB5BJd3iOnAXCrYLJTitWeGZlqzwe55VtsZqNnE,456
299
294
  airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py,sha256=2gqececTxxUqO6aIkVNNXADg48Px5EHUwnXHL9KiPT8,4188
@@ -304,13 +299,13 @@ airbyte_cdk/sources/streams/http/http.py,sha256=0uariNq8OFnlX7iqOHwBhecxA-Hfd5hS
304
299
  airbyte_cdk/sources/streams/http/http_client.py,sha256=tDE0ROtxjGMVphvsw8INvGMtZ97hIF-v47pZ3jIyiwc,23011
305
300
  airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
306
301
  airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
307
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=cM5CM1mnbTEMiY6gKHblGXr9KTS5VEziGoc-TXC302k,18791
302
+ airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=P9U8vtcrZ3m0InSG2W0H4gTYTxjQxkIe6mhF9xvO8Ug,18824
308
303
  airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=Y3n7J-sk5yGjv_OxtY6Z6k0PEsFZmtIRi-x0KCbaHdA,1010
309
304
  airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=C2j2uVfi9d-3KgHO3NGxIiFdfASjHOtsd6g_LWPYOAs,20311
310
305
  airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
311
306
  airbyte_cdk/sources/streams/permissions/identities_stream.py,sha256=9O9k6k18Xm3Zsiw_vnI_jsHXfMCQiek6V-jMkJJLxn8,2621
312
307
  airbyte_cdk/sources/streams/utils/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
313
- airbyte_cdk/sources/types.py,sha256=aFPGI4t2K1vHz2oFSUIYUyDN7kw-vcYq4D7aD2zgfAU,5128
308
+ airbyte_cdk/sources/types.py,sha256=x5Rkfpottg46qgr1-g_O4kN6v0Fd0sWHttdYsftyo7w,5148
314
309
  airbyte_cdk/sources/utils/__init__.py,sha256=TTN6VUxVy6Is8BhYQZR5pxJGQh8yH4duXh4O1TiMiEY,118
315
310
  airbyte_cdk/sources/utils/casing.py,sha256=QC-gV1O4e8DR4-bhdXieUPKm_JamzslVyfABLYYRSXA,256
316
311
  airbyte_cdk/sources/utils/record_helper.py,sha256=jeB0mucudzna7Zvj-pCBbwFrbLJ36SlAWZTh5O4Fb9Y,2168
@@ -352,7 +347,7 @@ airbyte_cdk/utils/datetime_format_inferrer.py,sha256=Ne2cpk7Tx3eZDEW2Q3O7jnNOY9g
352
347
  airbyte_cdk/utils/datetime_helpers.py,sha256=8mqzZ67Or2PBp7tLtrhh6XFv4wFzYsjCL_DOQJRaftI,17751
353
348
  airbyte_cdk/utils/event_timing.py,sha256=aiuFmPU80buLlNdKq4fDTEqqhEIelHPF6AalFGwY8as,2557
354
349
  airbyte_cdk/utils/is_cloud_environment.py,sha256=DayV32Irh-SdnJ0MnjvstwCJ66_l5oEsd8l85rZtHoc,574
355
- airbyte_cdk/utils/mapping_helpers.py,sha256=imUTULHmZ1Ks-MRMRLIVqHCX1eJi_j6tFQrYsKIKtM4,5967
350
+ airbyte_cdk/utils/mapping_helpers.py,sha256=nWjOpnz_5QE9tY9-GtSWMPvYQL95kDD6k8KwwjUmrh0,6526
356
351
  airbyte_cdk/utils/message_utils.py,sha256=OTzbkwN7AdMDA3iKYq1LKwfPFxpyEDfdgEF9BED3dkU,1366
357
352
  airbyte_cdk/utils/oneof_option_config.py,sha256=N8EmWdYdwt0FM7fuShh6H8nj_r4KEL9tb2DJJtwsPow,1180
358
353
  airbyte_cdk/utils/print_buffer.py,sha256=PhMOi0C4Z91kWKrSvCQXcp8qRh1uCimpIdvrg6voZIA,2810
@@ -361,9 +356,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
361
356
  airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
362
357
  airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
363
358
  airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
364
- airbyte_cdk-6.37.0.dev1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
365
- airbyte_cdk-6.37.0.dev1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
366
- airbyte_cdk-6.37.0.dev1.dist-info/METADATA,sha256=9grGzb0aDmQhE2MU2nXEKIYJFWcdpJpMTrSW5kTML3M,6015
367
- airbyte_cdk-6.37.0.dev1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
368
- airbyte_cdk-6.37.0.dev1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
369
- airbyte_cdk-6.37.0.dev1.dist-info/RECORD,,
359
+ airbyte_cdk-6.37.2.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
360
+ airbyte_cdk-6.37.2.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
361
+ airbyte_cdk-6.37.2.dist-info/METADATA,sha256=FWmhFRUtj2Ro3GXW1iMMP-ujCoO5TAio7TKotvadR9A,6013
362
+ airbyte_cdk-6.37.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
363
+ airbyte_cdk-6.37.2.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
364
+ airbyte_cdk-6.37.2.dist-info/RECORD,,
@@ -1,136 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- from dataclasses import dataclass
6
- from typing import Any, Iterable, Mapping, Optional
7
-
8
- from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
9
- from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
10
-
11
-
12
- @dataclass
13
- class GroupingPartitionRouter(PartitionRouter):
14
- """
15
- A partition router that groups partitions from an underlying partition router into batches of a specified size.
16
- This is useful for APIs that support filtering by multiple partition keys in a single request.
17
-
18
- Attributes:
19
- group_size (int): The number of partitions to include in each group.
20
- underlying_partition_router (PartitionRouter): The partition router whose output will be grouped.
21
- deduplicate (bool): If True, ensures unique partitions within each group by removing duplicates based on the partition key.
22
- config (Config): The connector configuration.
23
- parameters (Mapping[str, Any]): Additional parameters for interpolation and configuration.
24
- """
25
-
26
- group_size: int
27
- underlying_partition_router: PartitionRouter
28
- config: Config
29
- deduplicate: bool = True
30
-
31
- def stream_slices(self) -> Iterable[StreamSlice]:
32
- """
33
- Lazily groups partitions from the underlying partition router into batches of size `group_size`.
34
-
35
- This method processes partitions one at a time from the underlying router, maintaining a batch buffer.
36
- When the buffer reaches `group_size` or the underlying router is exhausted, it yields a grouped slice.
37
- If deduplication is enabled, it tracks seen partition keys to ensure uniqueness within the current batch.
38
-
39
- Yields:
40
- Iterable[StreamSlice]: An iterable of StreamSlice objects, where each slice contains a batch of partition values.
41
- """
42
- batch = []
43
- seen_keys = set()
44
-
45
- # Iterate over partitions lazily from the underlying router
46
- for partition in self.underlying_partition_router.stream_slices():
47
- # Extract the partition key (assuming single key-value pair, e.g., {"board_ids": value})
48
- key = next(iter(partition.partition.values()), None)
49
-
50
- # Skip duplicates if deduplication is enabled
51
- if self.deduplicate and key in seen_keys:
52
- continue
53
-
54
- # Add partition to the batch
55
- batch.append(partition)
56
- if self.deduplicate:
57
- seen_keys.add(key)
58
-
59
- # Yield the batch when it reaches the group_size
60
- if len(batch) == self.group_size:
61
- yield self._create_grouped_slice(batch)
62
- batch = [] # Reset the batch
63
-
64
- # Yield any remaining partitions if the batch isn't empty
65
- if batch:
66
- yield self._create_grouped_slice(batch)
67
-
68
- def _create_grouped_slice(self, batch: list[StreamSlice]) -> StreamSlice:
69
- """
70
- Creates a grouped StreamSlice from a batch of partitions, aggregating extra fields into a dictionary with list values.
71
-
72
- Args:
73
- batch (list[StreamSlice]): A list of StreamSlice objects to group.
74
-
75
- Returns:
76
- StreamSlice: A single StreamSlice with combined partition and extra field values.
77
- """
78
- # Combine partition values into a single dict with lists
79
- grouped_partition = {
80
- key: [p.partition.get(key) for p in batch] for key in batch[0].partition.keys()
81
- }
82
-
83
- # Aggregate extra fields into a dict with list values
84
- extra_fields_dict = (
85
- {
86
- key: [p.extra_fields.get(key) for p in batch]
87
- for key in set().union(*(p.extra_fields.keys() for p in batch if p.extra_fields))
88
- }
89
- if any(p.extra_fields for p in batch)
90
- else {}
91
- )
92
- return StreamSlice(
93
- partition=grouped_partition,
94
- cursor_slice={}, # Cursor is managed by the underlying router or incremental sync
95
- extra_fields=extra_fields_dict,
96
- )
97
-
98
- def get_request_params(
99
- self,
100
- stream_state: Optional[StreamState] = None,
101
- stream_slice: Optional[StreamSlice] = None,
102
- next_page_token: Optional[Mapping[str, Any]] = None,
103
- ) -> Mapping[str, Any]:
104
- return {}
105
-
106
- def get_request_headers(
107
- self,
108
- stream_state: Optional[StreamState] = None,
109
- stream_slice: Optional[StreamSlice] = None,
110
- next_page_token: Optional[Mapping[str, Any]] = None,
111
- ) -> Mapping[str, Any]:
112
- return {}
113
-
114
- def get_request_body_data(
115
- self,
116
- stream_state: Optional[StreamState] = None,
117
- stream_slice: Optional[StreamSlice] = None,
118
- next_page_token: Optional[Mapping[str, Any]] = None,
119
- ) -> Mapping[str, Any]:
120
- return {}
121
-
122
- def get_request_body_json(
123
- self,
124
- stream_state: Optional[StreamState] = None,
125
- stream_slice: Optional[StreamSlice] = None,
126
- next_page_token: Optional[Mapping[str, Any]] = None,
127
- ) -> Mapping[str, Any]:
128
- return {}
129
-
130
- def set_initial_state(self, stream_state: StreamState) -> None:
131
- """Delegate state initialization to the underlying partition router."""
132
- self.underlying_partition_router.set_initial_state(stream_state)
133
-
134
- def get_stream_state(self) -> Optional[Mapping[str, StreamState]]:
135
- """Delegate state retrieval to the underlying partition router."""
136
- return self.underlying_partition_router.get_stream_state()
@@ -1,3 +0,0 @@
1
- #
2
- # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3
- #
@@ -1,61 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- from abc import ABC, abstractmethod
6
- from typing import Generic, Iterable, Optional, TypeVar
7
-
8
- from airbyte_cdk.connector import TConfig
9
- from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type
10
- from airbyte_cdk.sources.embedded.catalog import (
11
- create_configured_catalog,
12
- get_stream,
13
- get_stream_names,
14
- )
15
- from airbyte_cdk.sources.embedded.runner import SourceRunner
16
- from airbyte_cdk.sources.embedded.tools import get_defined_id
17
- from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
18
-
19
- TOutput = TypeVar("TOutput")
20
-
21
-
22
- class BaseEmbeddedIntegration(ABC, Generic[TConfig, TOutput]):
23
- def __init__(self, runner: SourceRunner[TConfig], config: TConfig):
24
- check_config_against_spec_or_exit(config, runner.spec())
25
-
26
- self.source = runner
27
- self.config = config
28
-
29
- self.last_state: Optional[AirbyteStateMessage] = None
30
-
31
- @abstractmethod
32
- def _handle_record(self, record: AirbyteRecordMessage, id: Optional[str]) -> Optional[TOutput]:
33
- """
34
- Turn an Airbyte record into the appropriate output type for the integration.
35
- """
36
- pass
37
-
38
- def _load_data(
39
- self, stream_name: str, state: Optional[AirbyteStateMessage] = None
40
- ) -> Iterable[TOutput]:
41
- catalog = self.source.discover(self.config)
42
- stream = get_stream(catalog, stream_name)
43
- if not stream:
44
- raise ValueError(
45
- f"Stream {stream_name} not found, the following streams are available: {', '.join(get_stream_names(catalog))}"
46
- )
47
- if SyncMode.incremental not in stream.supported_sync_modes:
48
- configured_catalog = create_configured_catalog(stream, sync_mode=SyncMode.full_refresh)
49
- else:
50
- configured_catalog = create_configured_catalog(stream, sync_mode=SyncMode.incremental)
51
-
52
- for message in self.source.read(self.config, configured_catalog, state):
53
- if message.type == Type.RECORD:
54
- output = self._handle_record(
55
- message.record,
56
- get_defined_id(stream, message.record.data), # type: ignore[union-attr, arg-type]
57
- )
58
- if output:
59
- yield output
60
- elif message.type is Type.STATE and message.state:
61
- self.last_state = message.state
@@ -1,57 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- from typing import List, Optional
6
-
7
- from airbyte_cdk.models import (
8
- AirbyteCatalog,
9
- AirbyteStream,
10
- ConfiguredAirbyteCatalog,
11
- ConfiguredAirbyteStream,
12
- DestinationSyncMode,
13
- SyncMode,
14
- )
15
- from airbyte_cdk.sources.embedded.tools import get_first
16
-
17
-
18
- def get_stream(catalog: AirbyteCatalog, stream_name: str) -> Optional[AirbyteStream]:
19
- return get_first(catalog.streams, lambda s: s.name == stream_name)
20
-
21
-
22
- def get_stream_names(catalog: AirbyteCatalog) -> List[str]:
23
- return [stream.name for stream in catalog.streams]
24
-
25
-
26
- def to_configured_stream(
27
- stream: AirbyteStream,
28
- sync_mode: SyncMode = SyncMode.full_refresh,
29
- destination_sync_mode: DestinationSyncMode = DestinationSyncMode.append,
30
- cursor_field: Optional[List[str]] = None,
31
- primary_key: Optional[List[List[str]]] = None,
32
- ) -> ConfiguredAirbyteStream:
33
- return ConfiguredAirbyteStream(
34
- stream=stream,
35
- sync_mode=sync_mode,
36
- destination_sync_mode=destination_sync_mode,
37
- cursor_field=cursor_field,
38
- primary_key=primary_key,
39
- )
40
-
41
-
42
- def to_configured_catalog(
43
- configured_streams: List[ConfiguredAirbyteStream],
44
- ) -> ConfiguredAirbyteCatalog:
45
- return ConfiguredAirbyteCatalog(streams=configured_streams)
46
-
47
-
48
- def create_configured_catalog(
49
- stream: AirbyteStream, sync_mode: SyncMode = SyncMode.full_refresh
50
- ) -> ConfiguredAirbyteCatalog:
51
- configured_streams = [
52
- to_configured_stream(
53
- stream, sync_mode=sync_mode, primary_key=stream.source_defined_primary_key
54
- )
55
- ]
56
-
57
- return to_configured_catalog(configured_streams)
@@ -1,57 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
-
6
- import logging
7
- from abc import ABC, abstractmethod
8
- from typing import Generic, Iterable, Optional
9
-
10
- from airbyte_cdk.connector import TConfig
11
- from airbyte_cdk.models import (
12
- AirbyteCatalog,
13
- AirbyteMessage,
14
- AirbyteStateMessage,
15
- ConfiguredAirbyteCatalog,
16
- ConnectorSpecification,
17
- )
18
- from airbyte_cdk.sources.source import Source
19
-
20
-
21
- class SourceRunner(ABC, Generic[TConfig]):
22
- @abstractmethod
23
- def spec(self) -> ConnectorSpecification:
24
- pass
25
-
26
- @abstractmethod
27
- def discover(self, config: TConfig) -> AirbyteCatalog:
28
- pass
29
-
30
- @abstractmethod
31
- def read(
32
- self,
33
- config: TConfig,
34
- catalog: ConfiguredAirbyteCatalog,
35
- state: Optional[AirbyteStateMessage],
36
- ) -> Iterable[AirbyteMessage]:
37
- pass
38
-
39
-
40
- class CDKRunner(SourceRunner[TConfig]):
41
- def __init__(self, source: Source, name: str):
42
- self._source = source
43
- self._logger = logging.getLogger(name)
44
-
45
- def spec(self) -> ConnectorSpecification:
46
- return self._source.spec(self._logger)
47
-
48
- def discover(self, config: TConfig) -> AirbyteCatalog:
49
- return self._source.discover(self._logger, config)
50
-
51
- def read(
52
- self,
53
- config: TConfig,
54
- catalog: ConfiguredAirbyteCatalog,
55
- state: Optional[AirbyteStateMessage],
56
- ) -> Iterable[AirbyteMessage]:
57
- return self._source.read(self._logger, config, catalog, state=[state] if state else [])
@@ -1,27 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- from typing import Any, Callable, Dict, Iterable, Optional
6
-
7
- import dpath
8
-
9
- from airbyte_cdk.models import AirbyteStream
10
-
11
-
12
- def get_first(
13
- iterable: Iterable[Any], predicate: Callable[[Any], bool] = lambda m: True
14
- ) -> Optional[Any]:
15
- return next(filter(predicate, iterable), None)
16
-
17
-
18
- def get_defined_id(stream: AirbyteStream, data: Dict[str, Any]) -> Optional[str]:
19
- if not stream.source_defined_primary_key:
20
- return None
21
- primary_key = []
22
- for key in stream.source_defined_primary_key:
23
- try:
24
- primary_key.append(str(dpath.get(data, key)))
25
- except KeyError:
26
- primary_key.append("__not_found__")
27
- return "_".join(primary_key)