airbyte-cdk 6.45.0.post24.dev14387467928__py3-none-any.whl → 6.45.1.post42.dev14452300548__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/connector_builder/main.py +3 -3
- airbyte_cdk/models/__init__.py +1 -0
- airbyte_cdk/models/airbyte_protocol.py +1 -3
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +1 -1
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +3 -3
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +8 -0
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +36 -0
- airbyte_cdk/sources/declarative/extractors/record_selector.py +6 -1
- airbyte_cdk/sources/declarative/interpolation/filters.py +49 -2
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +31 -0
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +39 -1
- airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +9 -4
- airbyte_cdk/sources/file_based/file_based_source.py +3 -3
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +38 -15
- airbyte_cdk/sources/file_based/file_record_data.py +22 -0
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +1 -1
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +8 -15
- airbyte_cdk/sources/file_based/schema_helpers.py +9 -1
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +6 -15
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +15 -38
- airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +1 -3
- airbyte_cdk/sources/streams/concurrent/default_stream.py +3 -0
- airbyte_cdk/sources/types.py +11 -2
- airbyte_cdk/sources/utils/files_directory.py +15 -0
- airbyte_cdk/sources/utils/record_helper.py +8 -8
- airbyte_cdk/sql/shared/sql_processor.py +8 -9
- airbyte_cdk/test/entrypoint_wrapper.py +0 -4
- airbyte_cdk/test/mock_http/response_builder.py +8 -0
- {airbyte_cdk-6.45.0.post24.dev14387467928.dist-info → airbyte_cdk-6.45.1.post42.dev14452300548.dist-info}/METADATA +2 -2
- {airbyte_cdk-6.45.0.post24.dev14387467928.dist-info → airbyte_cdk-6.45.1.post42.dev14452300548.dist-info}/RECORD +35 -46
- airbyte_cdk/models/file_transfer_record_message.py +0 -13
- airbyte_cdk/test/declarative/__init__.py +0 -6
- airbyte_cdk/test/declarative/models/__init__.py +0 -7
- airbyte_cdk/test/declarative/models/scenario.py +0 -74
- airbyte_cdk/test/declarative/test_suites/__init__.py +0 -24
- airbyte_cdk/test/declarative/test_suites/connector_base.py +0 -202
- airbyte_cdk/test/declarative/test_suites/declarative_sources.py +0 -48
- airbyte_cdk/test/declarative/test_suites/destination_base.py +0 -12
- airbyte_cdk/test/declarative/test_suites/source_base.py +0 -129
- airbyte_cdk/test/declarative/utils/__init__.py +0 -0
- airbyte_cdk/test/declarative/utils/job_runner.py +0 -128
- airbyte_cdk/test/fixtures/__init__.py +0 -0
- airbyte_cdk/test/fixtures/auto.py +0 -14
- airbyte_cdk/test/pytest_config/plugin.py +0 -46
- {airbyte_cdk-6.45.0.post24.dev14387467928.dist-info → airbyte_cdk-6.45.1.post42.dev14452300548.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.45.0.post24.dev14387467928.dist-info → airbyte_cdk-6.45.1.post42.dev14452300548.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.45.0.post24.dev14387467928.dist-info → airbyte_cdk-6.45.1.post42.dev14452300548.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.45.0.post24.dev14387467928.dist-info → airbyte_cdk-6.45.1.post42.dev14452300548.dist-info}/entry_points.txt +0 -0
@@ -8,7 +8,7 @@ airbyte_cdk/connector.py,sha256=bO23kdGRkl8XKFytOgrrWFc_VagteTHVEF6IsbizVkM,4224
|
|
8
8
|
airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz52ftyNRnBw,1665
|
9
9
|
airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
10
10
|
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=gwFpZHmvFrMB9NMlRuRfq0KEkbPaFucCB8k1DL20Wmw,5769
|
11
|
-
airbyte_cdk/connector_builder/main.py,sha256=
|
11
|
+
airbyte_cdk/connector_builder/main.py,sha256=j1pP5N8RsnvQZ4iYxhLdLEHsJ5Ui7IVFBUi6wYMGBkM,3839
|
12
12
|
airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
|
13
13
|
airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
|
14
14
|
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=Iczn-_iczS2CaIAunWwyFcX0uLTra8Wh9JVfzm1Gfxo,26765
|
@@ -29,17 +29,16 @@ airbyte_cdk/destinations/vector_db_based/writer.py,sha256=nZ00xPiohElJmYktEZZIhr
|
|
29
29
|
airbyte_cdk/entrypoint.py,sha256=NRJv5BNZRSUEVTmNBa9N7ih6fW5sg4DwL0nkB9kI99Y,18570
|
30
30
|
airbyte_cdk/exception_handler.py,sha256=D_doVl3Dt60ASXlJsfviOCswxGyKF2q0RL6rif3fNks,2013
|
31
31
|
airbyte_cdk/logger.py,sha256=1cURbvawbunCAV178q-XhTHcbAQZTSf07WhU7U9AXWU,3744
|
32
|
-
airbyte_cdk/models/__init__.py,sha256=
|
33
|
-
airbyte_cdk/models/airbyte_protocol.py,sha256=
|
32
|
+
airbyte_cdk/models/__init__.py,sha256=Et9wJWs5VOWynGbb-3aJRhsdAHAiLkNNLxdwqJAuqkw,2114
|
33
|
+
airbyte_cdk/models/airbyte_protocol.py,sha256=oZdKsZ7yPjUt9hvxdWNpxCtgjSV2RWhf4R9Np03sqyY,3613
|
34
34
|
airbyte_cdk/models/airbyte_protocol_serializers.py,sha256=s6SaFB2CMrG_7jTQGn_fhFbQ1FUxhCxf5kq2RWGHMVI,1749
|
35
|
-
airbyte_cdk/models/file_transfer_record_message.py,sha256=J-E-43KOmUFdpsjeKlEfNnnZRSB-Gb5AGZjonR25Drc,323
|
36
35
|
airbyte_cdk/models/well_known_types.py,sha256=EquepbisGPuCSrs_D7YVVnMR9-ShhUr21wnFz3COiJs,156
|
37
36
|
airbyte_cdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
38
37
|
airbyte_cdk/sources/__init__.py,sha256=45J83QsFH3Wky3sVapZWg4C58R_i1thm61M06t2c1AQ,1156
|
39
38
|
airbyte_cdk/sources/abstract_source.py,sha256=50vxEBRByiNhT4WJkiFvgM-C6PWqKSJgvuNC_aeg2cw,15547
|
40
39
|
airbyte_cdk/sources/concurrent_source/__init__.py,sha256=3D_RJsxQfiLboSCDdNei1Iv-msRp3DXsas6E9kl7dXc,386
|
41
|
-
airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py,sha256=
|
42
|
-
airbyte_cdk/sources/concurrent_source/concurrent_source.py,sha256=
|
40
|
+
airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py,sha256=P_GA5QayzehCf0ksUbEbGoNixBnauzsepv-0ICzhH4w,12691
|
41
|
+
airbyte_cdk/sources/concurrent_source/concurrent_source.py,sha256=P8B6EcLKaSstfAD9kDZsTJ0q8vRmdFrxLt-zOA5_By0,7737
|
43
42
|
airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py,sha256=f9PIRPWn2tXu0-bxVeYHL2vYdqCzZ_kgpHg5_Ep-cfQ,6103
|
44
43
|
airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py,sha256=z1t-rAZBsqVidv2fpUlPHE9JgyXsITuGk4AMu96mXSQ,696
|
45
44
|
airbyte_cdk/sources/concurrent_source/stream_thread_exception.py,sha256=-q6mG2145HKQ28rZGD1bUmjPlIZ1S7-Yhewl8Ntu6xI,764
|
@@ -67,11 +66,11 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=QeExVmpSYjr_CnghHu
|
|
67
66
|
airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
|
68
67
|
airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
|
69
68
|
airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
|
70
|
-
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=
|
69
|
+
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=nqqcSe2FeycTpYz3KsFomeOzRWxGYc2G5YfFNQrMKlI,28113
|
71
70
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
72
71
|
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
|
73
72
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
74
|
-
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=
|
73
|
+
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=OCZDuyHSoubti45_LHfFHyPyZDjeFBhY1CymG9_buBk,159887
|
75
74
|
airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
|
76
75
|
airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
|
77
76
|
airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
|
@@ -89,7 +88,7 @@ airbyte_cdk/sources/declarative/extractors/dpath_extractor.py,sha256=wR4Ol4MG2lt
|
|
89
88
|
airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8VwkjS0xD_u350Km3SiYP7hpOOgiLg5o,1169
|
90
89
|
airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
|
91
90
|
airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
|
92
|
-
airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=
|
91
|
+
airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=Xg4e0exJ5Tq8I346uD2-HjpCsAUFLgPPcNKa0UoHjV8,7178
|
93
92
|
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=WJyA2OYIEgFpVP5Y3o0tIj69AV6IKkn9B16MeXaEItI,6513
|
94
93
|
airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
|
95
94
|
airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
|
@@ -101,7 +100,7 @@ airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py,sha256=9IAJT
|
|
101
100
|
airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py,sha256=2YBOA2NnwAeIKlIhSwUB_W-FaGnPcmrG_liY7b4mV2Y,8365
|
102
101
|
airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py,sha256=10LFv1QPM-agVKl6eaANmEBOfd7gZgBrkoTcMggsieQ,4809
|
103
102
|
airbyte_cdk/sources/declarative/interpolation/__init__.py,sha256=Kh7FxhfetyNVDnAQ9zSxNe4oUbb8CvoW7Mqz7cs2iPg,437
|
104
|
-
airbyte_cdk/sources/declarative/interpolation/filters.py,sha256=
|
103
|
+
airbyte_cdk/sources/declarative/interpolation/filters.py,sha256=cYap5zzOxIJWCLIfbkNlpyfUhjZ8FklLroIG4WGzYVs,5537
|
105
104
|
airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py,sha256=8F3ntT_Mfo8cO9n6dCq8rTfJIpfKmzRCsVtVdhzaoGc,1964
|
106
105
|
airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py,sha256=h36RIng4GZ9v4o_fRmgJjTNOtWmhK7NOILU1oSKPE4Q,2083
|
107
106
|
airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha256=vjwvkLk7_l6YDcFClwjCMcTleRjQBh7-dzny7PUaoG8,1857
|
@@ -114,13 +113,13 @@ airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW
|
|
114
113
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
|
115
114
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
116
115
|
airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
|
117
|
-
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=
|
116
|
+
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=EYPTtYTxNm_u-8OacYF6d4ce0PdV6jzYnxTmpjZOu_g,113491
|
118
117
|
airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
119
118
|
airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
|
120
119
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
|
121
120
|
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=4C15MKV-zOrMVQAm4FyohDsrJUBCSpMv5tZw0SK3aeI,9685
|
122
121
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
|
123
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
122
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=2mB379Yg7oD11j6HkuGq2OyOgmE2Sc4itPWo9XhI_hs,160128
|
124
123
|
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
|
125
124
|
airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
|
126
125
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
@@ -179,6 +178,7 @@ airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=d
|
|
179
178
|
airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
|
180
179
|
airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=nQepwG_RfW53sgwvK5dLPqfCx0VjsQ83nYoPjBMAaLM,527
|
181
180
|
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=6oZtnCHm9NdDvjTSrVwPQOXGSdETSIR7eWH2vFjM7jI,4855
|
181
|
+
airbyte_cdk/sources/declarative/retrievers/file_uploader.py,sha256=Om8vaDHix_1EXxEDVpRP57R94KmBmzJJLWtgxDJ5P8E,3363
|
182
182
|
airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
|
183
183
|
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=Ixs_byM-DMfYiYB-iOa97yH4WGAvxgTdTYeAsskHdYM,31134
|
184
184
|
airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
|
@@ -190,7 +190,7 @@ airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLn
|
|
190
190
|
airbyte_cdk/sources/declarative/spec/__init__.py,sha256=H0UwoRhgucbKBIzg85AXrifybVmfpwWpPdy22vZKVuo,141
|
191
191
|
airbyte_cdk/sources/declarative/spec/spec.py,sha256=ODSNUgkDOhnLQnwLjgSaME6R3kNeywjROvbNrWEnsgU,1876
|
192
192
|
airbyte_cdk/sources/declarative/stream_slicers/__init__.py,sha256=sI9vhc95RwJYOnA0VKjcbtKgFcmAbWjhdWBXFbAijOs,176
|
193
|
-
airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=
|
193
|
+
airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=cjKGm4r438dd1GxrFHJ4aYrdzG2bkncnwaWxAwlXR3M,3585
|
194
194
|
airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
|
195
195
|
airbyte_cdk/sources/declarative/transformations/__init__.py,sha256=CPJ8TlMpiUmvG3624VYu_NfTzxwKcfBjM2Q2wJ7fkSA,919
|
196
196
|
airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=Eg1jQtRObgzxbtySTQs5uEZIjEklsoHFxYSPf78x6Ng,5420
|
@@ -222,27 +222,28 @@ airbyte_cdk/sources/file_based/discovery_policy/__init__.py,sha256=gl3ey6mZbyfra
|
|
222
222
|
airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py,sha256=dCfXX529Rd5rtopg4VeEgTPJjFtqjtjzPq6LCw18Wt0,605
|
223
223
|
airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py,sha256=-xujTidtrq6HC00WKbjQh1CZdT5LMuzkp5BLjqDmfTY,1007
|
224
224
|
airbyte_cdk/sources/file_based/exceptions.py,sha256=WP0qkG6fpWoBpOyyicgp5YNE393VWyegq5qSy0v4QtM,7362
|
225
|
-
airbyte_cdk/sources/file_based/file_based_source.py,sha256=
|
225
|
+
airbyte_cdk/sources/file_based/file_based_source.py,sha256=Xg8OYWnGc-OcVBglvS08uwAWGWHBhEqsBnyODIkOK-4,20051
|
226
226
|
airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py,sha256=4e7FXqQ9hueacexC0SyrZyjF8oREYHza8pKF9CgKbD8,5050
|
227
|
-
airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=
|
227
|
+
airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=m70_TP1dlC54VWcL01uHSMSu-VZ5SEzsTJ9c43ycISk,7639
|
228
|
+
airbyte_cdk/sources/file_based/file_record_data.py,sha256=FqqVLzJlkT6ajW1ncoB8K99CvS8jZThafNzjsGGBQhM,396
|
228
229
|
airbyte_cdk/sources/file_based/file_types/__init__.py,sha256=blCLn0-2LC-ZdgcNyDEhqM2RiUvEjEBh-G4-t32ZtuM,1268
|
229
|
-
airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=
|
230
|
+
airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=USEYqiICXBWpDV443VtNOCmUA-GINzY_Zah74_5w3qQ,10860
|
230
231
|
airbyte_cdk/sources/file_based/file_types/csv_parser.py,sha256=QlCXB-ry3np67Q_VerQEPoWDOTcPTB6Go4ydZxY9ae4,20445
|
231
232
|
airbyte_cdk/sources/file_based/file_types/excel_parser.py,sha256=BeplCq0hmojELU6bZCvvpRLpQ9us81TqbGYwrhd3INo,7188
|
232
|
-
airbyte_cdk/sources/file_based/file_types/file_transfer.py,sha256=
|
233
|
+
airbyte_cdk/sources/file_based/file_types/file_transfer.py,sha256=5l2Jo6bp6neDmgM427PrZMZeqU0hCIZVWnzUZ_7BT10,1100
|
233
234
|
airbyte_cdk/sources/file_based/file_types/file_type_parser.py,sha256=JgpH21PrbRqwK92BJklZWvh2TndA6xZ-eP1LPMo44oQ,2832
|
234
235
|
airbyte_cdk/sources/file_based/file_types/jsonl_parser.py,sha256=GwyNyxmST4RX-XpXy7xVH0D-znYWWBmGv_pVAu95oHQ,5886
|
235
236
|
airbyte_cdk/sources/file_based/file_types/parquet_parser.py,sha256=XenFg5sJ-UBnIkSmsiNJRou11NO0zZXx-RXgPHMT2NA,10487
|
236
237
|
airbyte_cdk/sources/file_based/file_types/unstructured_parser.py,sha256=2TYOQl62FQPCa8otLbkDIk_j01EP3oWaKSfXGhCjCHg,19492
|
237
238
|
airbyte_cdk/sources/file_based/remote_file.py,sha256=yqRz93vPe8PBXLIMJ5W5u2JRlZRhg6sBrAjn3pPjJ8A,315
|
238
|
-
airbyte_cdk/sources/file_based/schema_helpers.py,sha256=
|
239
|
+
airbyte_cdk/sources/file_based/schema_helpers.py,sha256=rtyay5yy0khfZ9Fj4zZ512zBgNXnLXBR-8r85SUu9B8,9817
|
239
240
|
airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py,sha256=FkByIyEy56x2_awYnxGPqGaOp7zAzpAoRkPZHKySI9M,536
|
240
241
|
airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py,sha256=kjvX7nOmUALYd7HuZHilUzgJPZ-MnZ08mtvuBnt2tQ0,618
|
241
242
|
airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py,sha256=vjTlmYT_nqzY3DbT5xem7X-bwgA9RyXHoKFqiMO2URk,1728
|
242
243
|
airbyte_cdk/sources/file_based/stream/__init__.py,sha256=q_zmeOHHg0JK5j1YNSOIsyXGz-wlTl_0E8z5GKVAcVM,543
|
243
244
|
airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=9pQh3BHYcxm8CRC8XawfmBxL8O9HggpWwCCbX_ncINE,7509
|
244
245
|
airbyte_cdk/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
245
|
-
airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=
|
246
|
+
airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=1AIuAOHa_M6zN9l0eAWBHwhKl4fdP4-KlUMOMzTv11U,13525
|
246
247
|
airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py,sha256=Rx7TwjH8B7e0eee83Tlqxv1bWn-BVXOmlUAH7auM1uM,344
|
247
248
|
airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py,sha256=5dYZMLBEbvCyrCT89lCYdm2FdrLPLuxjdpQSVGP5o0w,1856
|
248
249
|
airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py,sha256=gRTL-9I3ejjQOpLKd6ixe9rB3kGlubCdhUt9ri6AdAI,14880
|
@@ -250,9 +251,9 @@ airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_c
|
|
250
251
|
airbyte_cdk/sources/file_based/stream/cursor/__init__.py,sha256=MhFB5hOo8sjwvCh8gangaymdg3EJWYt_72brFOZt068,191
|
251
252
|
airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py,sha256=om-x3gZFPgWDpi15S9RxZmR36VHnk8sytgN6LlBQhAw,1934
|
252
253
|
airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py,sha256=VGV7xLyBribuBMVrXtO1xqkWJD86bl7yhXtjnwLMohM,7051
|
253
|
-
airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=
|
254
|
+
airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=gzSN4cLywwzo_U1OdFS3Of_-4DRkUcX_j7Mv30MrxQs,17154
|
254
255
|
airbyte_cdk/sources/file_based/stream/identities_stream.py,sha256=FZH83Geoy3K3nwUk2VVNJERFcXUTnl-4XljjucUM23s,1893
|
255
|
-
airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py,sha256=
|
256
|
+
airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py,sha256=6KxqdD3-VvwxDTk7TtZ0M32fga4CI3qZ9IKdAkySpx0,3844
|
256
257
|
airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
|
257
258
|
airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
|
258
259
|
airbyte_cdk/sources/http_logger.py,sha256=H93kPAujHhPmXNX0JSFG3D-SL6yEFA5PtKot9Hu3TYA,1690
|
@@ -278,7 +279,7 @@ airbyte_cdk/sources/streams/concurrent/availability_strategy.py,sha256=4La5v2Uff
|
|
278
279
|
airbyte_cdk/sources/streams/concurrent/clamping.py,sha256=i26GVyui2ScEXSP-IP_61K2HaTp1-6lTlYHsZVYpuZA,3240
|
279
280
|
airbyte_cdk/sources/streams/concurrent/cursor.py,sha256=LFXbKBEMtNSVz_kZs9qydS9fPvzTU5wdgXRagRRJeHo,21388
|
280
281
|
airbyte_cdk/sources/streams/concurrent/cursor_types.py,sha256=ZyWLPpeLX1qXcP5MwS-wxK11IBMsnVPCw9zx8gA2_Ro,843
|
281
|
-
airbyte_cdk/sources/streams/concurrent/default_stream.py,sha256=
|
282
|
+
airbyte_cdk/sources/streams/concurrent/default_stream.py,sha256=3SBjFa1z955pSE_2qt1C7mAky-RKjOZeQDePbZkWYYs,3371
|
282
283
|
airbyte_cdk/sources/streams/concurrent/exceptions.py,sha256=JOZ446MCLpmF26r9KfS6OO_6rGjcjgJNZdcw6jccjEI,468
|
283
284
|
airbyte_cdk/sources/streams/concurrent/helpers.py,sha256=S6AW8TgIASCZ2UuUcQLE8OzgYUHWt2-KPOvNPwnQf-Q,1596
|
284
285
|
airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py,sha256=2t64b_z9cEPmlHZnjSiMTO8PEtEdiAJDG0JcYOtUqAE,3363
|
@@ -315,10 +316,11 @@ airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=C2j2uVfi9d
|
|
315
316
|
airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
|
316
317
|
airbyte_cdk/sources/streams/permissions/identities_stream.py,sha256=9O9k6k18Xm3Zsiw_vnI_jsHXfMCQiek6V-jMkJJLxn8,2621
|
317
318
|
airbyte_cdk/sources/streams/utils/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
318
|
-
airbyte_cdk/sources/types.py,sha256=
|
319
|
+
airbyte_cdk/sources/types.py,sha256=1oerHQpidFrIluUhWsM3-4Xst4wwUDqzkgtgZY0jjZU,5485
|
319
320
|
airbyte_cdk/sources/utils/__init__.py,sha256=TTN6VUxVy6Is8BhYQZR5pxJGQh8yH4duXh4O1TiMiEY,118
|
320
321
|
airbyte_cdk/sources/utils/casing.py,sha256=QC-gV1O4e8DR4-bhdXieUPKm_JamzslVyfABLYYRSXA,256
|
321
|
-
airbyte_cdk/sources/utils/
|
322
|
+
airbyte_cdk/sources/utils/files_directory.py,sha256=z8Dmr-wkL1sAqdwCST4MBUFAyMHPD2cJIzVdAuCynp8,391
|
323
|
+
airbyte_cdk/sources/utils/record_helper.py,sha256=7wL-pDYrBpcmZHa8ORtiSOqBZJEZI5hdl2dA1RYiatk,2029
|
322
324
|
airbyte_cdk/sources/utils/schema_helpers.py,sha256=bR3I70-e11S6B8r6VK-pthQXtcYrXojgXFvuK7lRrpg,8545
|
323
325
|
airbyte_cdk/sources/utils/slice_logger.py,sha256=qWWeFLAvigFz0b4O1_O3QDM1cy8PqZAMMgVPR2hEeb8,1778
|
324
326
|
airbyte_cdk/sources/utils/transform.py,sha256=0LOvIJg1vmg_70AiAVe-YHMr-LHrqEuxg9cm1BnYPDM,11725
|
@@ -332,30 +334,17 @@ airbyte_cdk/sql/exceptions.py,sha256=7_-K2c_trPy6kM89I2pwsrnVEtXqOspd9Eqrzf2KD2A
|
|
332
334
|
airbyte_cdk/sql/secrets.py,sha256=FRIafU5YbWzoK8jtAcfExwzMGdswMbs0OOo1O7Y5i-g,4345
|
333
335
|
airbyte_cdk/sql/shared/__init__.py,sha256=-BU9zpzwx7JxSlS7EmFuGmdB9jK_QhhEJUe5dxErrDw,334
|
334
336
|
airbyte_cdk/sql/shared/catalog_providers.py,sha256=qiahORhtN6qBUGHhSKmzE00uC4i6W8unyBKCj7Kw47s,5218
|
335
|
-
airbyte_cdk/sql/shared/sql_processor.py,sha256=
|
337
|
+
airbyte_cdk/sql/shared/sql_processor.py,sha256=1CwfC3fp9dWnHBpKtly7vGduf9ho_MahiwxGFcULG3Y,27687
|
336
338
|
airbyte_cdk/sql/types.py,sha256=XEIhRAo_ASd0kVLBkdLf5bHiRhNple-IJrC9TibcDdY,5880
|
337
339
|
airbyte_cdk/test/__init__.py,sha256=f_XdkOg4_63QT2k3BbKY34209lppwgw-svzfZstQEq4,199
|
338
340
|
airbyte_cdk/test/catalog_builder.py,sha256=-y05Cz1x0Dlk6oE9LSKhCozssV2gYBNtMdV5YYOPOtk,3015
|
339
|
-
airbyte_cdk/test/
|
340
|
-
airbyte_cdk/test/declarative/models/__init__.py,sha256=rXoywbDd-oqEhRQLuaEQIDxykMeawHjzzeN3XIpGQN8,132
|
341
|
-
airbyte_cdk/test/declarative/models/scenario.py,sha256=ySzQosxi2TRaw7JbQMFsuBieoH0ufXJhEY8Vu3DXtx4,2378
|
342
|
-
airbyte_cdk/test/declarative/test_suites/__init__.py,sha256=E1KY-xxKjkKXYaAVyCs96-ZMCCoDhR24s2eQhWWOx-c,769
|
343
|
-
airbyte_cdk/test/declarative/test_suites/connector_base.py,sha256=lT-fectu_9l-65RjQ7plE0X8kfB5CdOuY_tGwzJYtpI,6767
|
344
|
-
airbyte_cdk/test/declarative/test_suites/declarative_sources.py,sha256=4VPGzG8keyci0x2o6_jNqx7HcAuamumJoPafudH5w_o,1633
|
345
|
-
airbyte_cdk/test/declarative/test_suites/destination_base.py,sha256=L_l1gpNCkzfx-c7mzS1He5hTVbqR39OnfWdrYMglS7E,486
|
346
|
-
airbyte_cdk/test/declarative/test_suites/source_base.py,sha256=1VJ9KXM1nSodFKh2VKXc2cO010JfE-df2JoVGKasrmk,4381
|
347
|
-
airbyte_cdk/test/declarative/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
348
|
-
airbyte_cdk/test/declarative/utils/job_runner.py,sha256=AI10Nt5MEMbJt_zNNBatUPStvMEF1mstHzow-11ZVsE,4845
|
349
|
-
airbyte_cdk/test/entrypoint_wrapper.py,sha256=TyUmVJyIuGelAv6y8Wy_BnwqIRw_drjfZWKlroljCuQ,9951
|
350
|
-
airbyte_cdk/test/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
351
|
-
airbyte_cdk/test/fixtures/auto.py,sha256=wYKu1pBUJX60Bg-K2RwNXi11Txu1Rncc8WNlOvy9Zug,354
|
341
|
+
airbyte_cdk/test/entrypoint_wrapper.py,sha256=9XBii_YguQp0d8cykn3hy102FsJcwIBQzSB7co5ho0s,9802
|
352
342
|
airbyte_cdk/test/mock_http/__init__.py,sha256=jE5kC6CQ0OXkTqKhciDnNVZHesBFVIA2YvkdFGwva7k,322
|
353
343
|
airbyte_cdk/test/mock_http/matcher.py,sha256=4Qj8UnJKZIs-eodshryce3SN1Ayc8GZpBETmP6hTEyc,1446
|
354
344
|
airbyte_cdk/test/mock_http/mocker.py,sha256=XgsjMtVoeMpRELPyALgrkHFauH9H5irxrz1Kcxh2yFY,8013
|
355
345
|
airbyte_cdk/test/mock_http/request.py,sha256=tdB8cqk2vLgCDTOKffBKsM06llYs4ZecgtH6DKyx6yY,4112
|
356
346
|
airbyte_cdk/test/mock_http/response.py,sha256=s4-cQQqTtmeej0pQDWqmG0vUWpHS-93lIWMpW3zSVyU,662
|
357
|
-
airbyte_cdk/test/mock_http/response_builder.py,sha256=
|
358
|
-
airbyte_cdk/test/pytest_config/plugin.py,sha256=Z2RYpXOKTRsytb0pn0K-PvkQlQXeOppnEcX4IlBEZfI,1457
|
347
|
+
airbyte_cdk/test/mock_http/response_builder.py,sha256=F-v7ebftqGj7YVIMLKdodmU9U8Dq8aIyllWGo2NGwHc,8331
|
359
348
|
airbyte_cdk/test/state_builder.py,sha256=kLPql9lNzUJaBg5YYRLJlY_Hy5JLHJDVyKPMZMoYM44,946
|
360
349
|
airbyte_cdk/test/utils/__init__.py,sha256=Hu-1XT2KDoYjDF7-_ziDwv5bY3PueGjANOCbzeOegDg,57
|
361
350
|
airbyte_cdk/test/utils/data.py,sha256=CkCR1_-rujWNmPXFR1IXTMwx1rAl06wAyIKWpDcN02w,820
|
@@ -379,9 +368,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
379
368
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
380
369
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
381
370
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
382
|
-
airbyte_cdk-6.45.
|
383
|
-
airbyte_cdk-6.45.
|
384
|
-
airbyte_cdk-6.45.
|
385
|
-
airbyte_cdk-6.45.
|
386
|
-
airbyte_cdk-6.45.
|
387
|
-
airbyte_cdk-6.45.
|
371
|
+
airbyte_cdk-6.45.1.post42.dev14452300548.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
372
|
+
airbyte_cdk-6.45.1.post42.dev14452300548.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
373
|
+
airbyte_cdk-6.45.1.post42.dev14452300548.dist-info/METADATA,sha256=qy4fq1GiUd-OgGyr4GjZWkc699QSQRWZf6-u13sjMXc,6093
|
374
|
+
airbyte_cdk-6.45.1.post42.dev14452300548.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
375
|
+
airbyte_cdk-6.45.1.post42.dev14452300548.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
376
|
+
airbyte_cdk-6.45.1.post42.dev14452300548.dist-info/RECORD,,
|
@@ -1,13 +0,0 @@
|
|
1
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
-
|
3
|
-
from dataclasses import dataclass
|
4
|
-
from typing import Any, Dict, Optional
|
5
|
-
|
6
|
-
|
7
|
-
@dataclass
|
8
|
-
class AirbyteFileTransferRecordMessage:
|
9
|
-
stream: str
|
10
|
-
file: Dict[str, Any]
|
11
|
-
emitted_at: int
|
12
|
-
namespace: Optional[str] = None
|
13
|
-
data: Optional[Dict[str, Any]] = None
|
@@ -1,74 +0,0 @@
|
|
1
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
-
"""Run acceptance tests in PyTest.
|
3
|
-
|
4
|
-
These tests leverage the same `acceptance-test-config.yml` configuration files as the
|
5
|
-
acceptance tests in CAT, but they run in PyTest instead of CAT. This allows us to run
|
6
|
-
the acceptance tests in the same local environment as we are developing in, speeding
|
7
|
-
up iteration cycles.
|
8
|
-
"""
|
9
|
-
|
10
|
-
from __future__ import annotations
|
11
|
-
|
12
|
-
from pathlib import Path
|
13
|
-
from typing import Any, Literal, cast
|
14
|
-
|
15
|
-
import yaml
|
16
|
-
from pydantic import BaseModel
|
17
|
-
|
18
|
-
|
19
|
-
class ConnectorTestScenario(BaseModel):
|
20
|
-
"""Acceptance test instance, as a Pydantic model.
|
21
|
-
|
22
|
-
This class represents an acceptance test instance, which is a single test case
|
23
|
-
that can be run against a connector. It is used to deserialize and validate the
|
24
|
-
acceptance test configuration file.
|
25
|
-
"""
|
26
|
-
|
27
|
-
class AcceptanceTestExpectRecords(BaseModel):
|
28
|
-
path: Path
|
29
|
-
exact_order: bool = False
|
30
|
-
|
31
|
-
class AcceptanceTestFileTypes(BaseModel):
|
32
|
-
skip_test: bool
|
33
|
-
bypass_reason: str
|
34
|
-
|
35
|
-
config_path: Path | None = None
|
36
|
-
config_dict: dict[str, Any] | None = None
|
37
|
-
|
38
|
-
id: str | None = None
|
39
|
-
|
40
|
-
configured_catalog_path: Path | None = None
|
41
|
-
timeout_seconds: int | None = None
|
42
|
-
expect_records: AcceptanceTestExpectRecords | None = None
|
43
|
-
file_types: AcceptanceTestFileTypes | None = None
|
44
|
-
status: Literal["succeed", "failed"] | None = None
|
45
|
-
|
46
|
-
def get_config_dict(self) -> dict[str, Any]:
|
47
|
-
"""Return the config dictionary.
|
48
|
-
|
49
|
-
If a config dictionary has already been loaded, return it. Otherwise, load
|
50
|
-
the config file and return the dictionary.
|
51
|
-
"""
|
52
|
-
if self.config_dict:
|
53
|
-
return self.config_dict
|
54
|
-
|
55
|
-
if self.config_path:
|
56
|
-
return cast(dict[str, Any], yaml.safe_load(self.config_path.read_text()))
|
57
|
-
|
58
|
-
raise ValueError("No config dictionary or path provided.")
|
59
|
-
|
60
|
-
@property
|
61
|
-
def expect_exception(self) -> bool:
|
62
|
-
return self.status and self.status == "failed" or False
|
63
|
-
|
64
|
-
@property
|
65
|
-
def instance_name(self) -> str:
|
66
|
-
return self.config_path.stem if self.config_path else "Unnamed Scenario"
|
67
|
-
|
68
|
-
def __str__(self) -> str:
|
69
|
-
if self.id:
|
70
|
-
return f"'{self.id}' Test Scenario"
|
71
|
-
if self.config_path:
|
72
|
-
return f"'{self.config_path.name}' Test Scenario"
|
73
|
-
|
74
|
-
return f"'{hash(self)}' Test Scenario"
|
@@ -1,24 +0,0 @@
|
|
1
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
-
"""Declarative test suites.
|
3
|
-
|
4
|
-
Here we have base classes for a robust set of declarative connector test suites.
|
5
|
-
"""
|
6
|
-
|
7
|
-
from airbyte_cdk.test.declarative.test_suites.connector_base import (
|
8
|
-
ConnectorTestScenario,
|
9
|
-
generate_tests,
|
10
|
-
)
|
11
|
-
from airbyte_cdk.test.declarative.test_suites.declarative_sources import (
|
12
|
-
DeclarativeSourceTestSuite,
|
13
|
-
)
|
14
|
-
from airbyte_cdk.test.declarative.test_suites.destination_base import DestinationTestSuiteBase
|
15
|
-
from airbyte_cdk.test.declarative.test_suites.source_base import SourceTestSuiteBase
|
16
|
-
|
17
|
-
__all__ = [
|
18
|
-
"ConnectorTestScenario",
|
19
|
-
"ConnectorTestSuiteBase",
|
20
|
-
"DeclarativeSourceTestSuite",
|
21
|
-
"DestinationTestSuiteBase",
|
22
|
-
"SourceTestSuiteBase",
|
23
|
-
"generate_tests",
|
24
|
-
]
|
@@ -1,202 +0,0 @@
|
|
1
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
-
"""Base class for connector test suites."""
|
3
|
-
|
4
|
-
from __future__ import annotations
|
5
|
-
|
6
|
-
import abc
|
7
|
-
import functools
|
8
|
-
import inspect
|
9
|
-
import sys
|
10
|
-
from pathlib import Path
|
11
|
-
from typing import Any, Callable, Literal
|
12
|
-
|
13
|
-
import pytest
|
14
|
-
import yaml
|
15
|
-
from pydantic import BaseModel
|
16
|
-
from typing_extensions import override
|
17
|
-
|
18
|
-
from airbyte_cdk import Connector
|
19
|
-
from airbyte_cdk.models import (
|
20
|
-
AirbyteMessage,
|
21
|
-
Type,
|
22
|
-
)
|
23
|
-
from airbyte_cdk.sources import Source
|
24
|
-
from airbyte_cdk.sources.declarative.declarative_source import (
|
25
|
-
AbstractSource,
|
26
|
-
DeclarativeSource,
|
27
|
-
)
|
28
|
-
from airbyte_cdk.test import entrypoint_wrapper
|
29
|
-
from airbyte_cdk.test.declarative.models import (
|
30
|
-
ConnectorTestScenario,
|
31
|
-
)
|
32
|
-
from airbyte_cdk.test.declarative.utils.job_runner import run_test_job
|
33
|
-
|
34
|
-
ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
|
35
|
-
|
36
|
-
|
37
|
-
class JavaClass(str):
|
38
|
-
"""A string that represents a Java class."""
|
39
|
-
|
40
|
-
|
41
|
-
class DockerImage(str):
|
42
|
-
"""A string that represents a Docker image."""
|
43
|
-
|
44
|
-
|
45
|
-
class RunnableConnector(abc.ABC):
|
46
|
-
"""A connector that can be run in a test scenario."""
|
47
|
-
|
48
|
-
@abc.abstractmethod
|
49
|
-
def launch(cls, args: list[str] | None) -> None: ...
|
50
|
-
|
51
|
-
|
52
|
-
def generate_tests(metafunc) -> None:
|
53
|
-
"""
|
54
|
-
A helper for pytest_generate_tests hook.
|
55
|
-
|
56
|
-
If a test method (in a class subclassed from our base class)
|
57
|
-
declares an argument 'instance', this function retrieves the
|
58
|
-
'scenarios' attribute from the test class and parametrizes that
|
59
|
-
test with the values from 'scenarios'.
|
60
|
-
|
61
|
-
## Usage
|
62
|
-
|
63
|
-
```python
|
64
|
-
from airbyte_cdk.test.declarative.test_suites.connector_base import (
|
65
|
-
generate_tests,
|
66
|
-
ConnectorTestSuiteBase,
|
67
|
-
)
|
68
|
-
|
69
|
-
def pytest_generate_tests(metafunc):
|
70
|
-
generate_tests(metafunc)
|
71
|
-
|
72
|
-
class TestMyConnector(ConnectorTestSuiteBase):
|
73
|
-
...
|
74
|
-
|
75
|
-
```
|
76
|
-
"""
|
77
|
-
# Check if the test function requires an 'instance' argument
|
78
|
-
if "instance" in metafunc.fixturenames:
|
79
|
-
# Retrieve the test class
|
80
|
-
test_class = metafunc.cls
|
81
|
-
if test_class is None:
|
82
|
-
raise ValueError("Expected a class here.")
|
83
|
-
# Get the 'scenarios' attribute from the class
|
84
|
-
scenarios_attr = getattr(test_class, "get_scenarios", None)
|
85
|
-
if scenarios_attr is None:
|
86
|
-
raise ValueError(
|
87
|
-
f"Test class {test_class} does not have a 'scenarios' attribute. "
|
88
|
-
"Please define the 'scenarios' attribute in the test class."
|
89
|
-
)
|
90
|
-
|
91
|
-
scenarios = test_class.get_scenarios()
|
92
|
-
ids = [str(scenario) for scenario in scenarios]
|
93
|
-
metafunc.parametrize("instance", scenarios, ids=ids)
|
94
|
-
|
95
|
-
|
96
|
-
class ConnectorTestSuiteBase(abc.ABC):
|
97
|
-
"""Base class for connector test suites."""
|
98
|
-
|
99
|
-
acceptance_test_file_path = Path("./acceptance-test-config.json")
|
100
|
-
"""The path to the acceptance test config file.
|
101
|
-
|
102
|
-
By default, this is set to the `acceptance-test-config.json` file in
|
103
|
-
the root of the connector source directory.
|
104
|
-
"""
|
105
|
-
|
106
|
-
connector: type[Connector] | Path | JavaClass | DockerImage | None = None
|
107
|
-
"""The connector class or path to the connector to test."""
|
108
|
-
|
109
|
-
working_dir: Path | None = None
|
110
|
-
"""The root directory of the connector source code."""
|
111
|
-
|
112
|
-
@classmethod
|
113
|
-
def create_connector(
|
114
|
-
cls, scenario: ConnectorTestScenario
|
115
|
-
) -> Source | AbstractSource | DeclarativeSource | RunnableConnector:
|
116
|
-
"""Instantiate the connector class."""
|
117
|
-
raise NotImplementedError("Subclasses must implement this method.")
|
118
|
-
|
119
|
-
def run_test_scenario(
|
120
|
-
self,
|
121
|
-
verb: Literal["read", "check", "discover"],
|
122
|
-
test_scenario: ConnectorTestScenario,
|
123
|
-
*,
|
124
|
-
catalog: dict | None = None,
|
125
|
-
) -> entrypoint_wrapper.EntrypointOutput:
|
126
|
-
"""Run a test job from provided CLI args and return the result."""
|
127
|
-
return run_test_job(
|
128
|
-
self.create_connector(test_scenario),
|
129
|
-
verb,
|
130
|
-
test_instance=test_scenario,
|
131
|
-
catalog=catalog,
|
132
|
-
)
|
133
|
-
|
134
|
-
# Test Definitions
|
135
|
-
|
136
|
-
def test_check(
|
137
|
-
self,
|
138
|
-
instance: ConnectorTestScenario,
|
139
|
-
) -> None:
|
140
|
-
"""Run `connection` acceptance tests."""
|
141
|
-
result = self.run_test_scenario(
|
142
|
-
"check",
|
143
|
-
test_scenario=instance,
|
144
|
-
)
|
145
|
-
conn_status_messages: list[AirbyteMessage] = [
|
146
|
-
msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
|
147
|
-
] # noqa: SLF001 # Non-public API
|
148
|
-
assert len(conn_status_messages) == 1, (
|
149
|
-
"Expected exactly one CONNECTION_STATUS message. Got: \n" + "\n".join(result._messages)
|
150
|
-
)
|
151
|
-
|
152
|
-
@classmethod
|
153
|
-
@property
|
154
|
-
def acceptance_test_config_path(self) -> Path:
|
155
|
-
"""Get the path to the acceptance test config file.
|
156
|
-
|
157
|
-
Check vwd and parent directories of cwd for the config file, and return the first one found.
|
158
|
-
|
159
|
-
Give up if the config file is not found in any parent directory.
|
160
|
-
"""
|
161
|
-
current_dir = Path.cwd()
|
162
|
-
for parent_dir in current_dir.parents:
|
163
|
-
config_path = parent_dir / ACCEPTANCE_TEST_CONFIG
|
164
|
-
if config_path.exists():
|
165
|
-
return config_path
|
166
|
-
raise FileNotFoundError(
|
167
|
-
f"Acceptance test config file not found in any parent directory from : {Path.cwd()}"
|
168
|
-
)
|
169
|
-
|
170
|
-
@classmethod
|
171
|
-
def get_scenarios(
|
172
|
-
cls,
|
173
|
-
) -> list[ConnectorTestScenario]:
|
174
|
-
"""Get acceptance tests for a given category.
|
175
|
-
|
176
|
-
This has to be a separate function because pytest does not allow
|
177
|
-
parametrization of fixtures with arguments from the test class itself.
|
178
|
-
"""
|
179
|
-
category = "connection"
|
180
|
-
all_tests_config = yaml.safe_load(cls.acceptance_test_config_path.read_text())
|
181
|
-
if "acceptance_tests" not in all_tests_config:
|
182
|
-
raise ValueError(
|
183
|
-
f"Acceptance tests config not found in {cls.acceptance_test_config_path}."
|
184
|
-
f" Found only: {str(all_tests_config)}."
|
185
|
-
)
|
186
|
-
if category not in all_tests_config["acceptance_tests"]:
|
187
|
-
return []
|
188
|
-
if "tests" not in all_tests_config["acceptance_tests"][category]:
|
189
|
-
raise ValueError(f"No tests found for category {category}")
|
190
|
-
|
191
|
-
tests_scenarios = [
|
192
|
-
ConnectorTestScenario.model_validate(test)
|
193
|
-
for test in all_tests_config["acceptance_tests"][category]["tests"]
|
194
|
-
if "iam_role" not in test["config_path"]
|
195
|
-
]
|
196
|
-
working_dir = cls.working_dir or Path()
|
197
|
-
for test in tests_scenarios:
|
198
|
-
if test.config_path:
|
199
|
-
test.config_path = working_dir / test.config_path
|
200
|
-
if test.configured_catalog_path:
|
201
|
-
test.configured_catalog_path = working_dir / test.configured_catalog_path
|
202
|
-
return tests_scenarios
|
@@ -1,48 +0,0 @@
|
|
1
|
-
import os
|
2
|
-
from hashlib import md5
|
3
|
-
from pathlib import Path
|
4
|
-
from typing import Any, cast
|
5
|
-
|
6
|
-
import yaml
|
7
|
-
|
8
|
-
from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
|
9
|
-
ConcurrentDeclarativeSource,
|
10
|
-
)
|
11
|
-
from airbyte_cdk.test.declarative.models import ConnectorTestScenario
|
12
|
-
from airbyte_cdk.test.declarative.test_suites.source_base import (
|
13
|
-
SourceTestSuiteBase,
|
14
|
-
)
|
15
|
-
|
16
|
-
|
17
|
-
def md5_checksum(file_path: Path) -> str:
|
18
|
-
with open(file_path, "rb") as file:
|
19
|
-
return md5(file.read()).hexdigest()
|
20
|
-
|
21
|
-
|
22
|
-
class DeclarativeSourceTestSuite(SourceTestSuiteBase):
|
23
|
-
manifest_path = Path("manifest.yaml")
|
24
|
-
components_py_path: Path | None = None
|
25
|
-
|
26
|
-
def create_connector(
|
27
|
-
self, connector_test: ConnectorTestScenario
|
28
|
-
) -> ConcurrentDeclarativeSource:
|
29
|
-
"""Create a connector instance for the test suite."""
|
30
|
-
config = connector_test.get_config_dict()
|
31
|
-
# catalog = connector_test.get_catalog()
|
32
|
-
# state = connector_test.get_state()
|
33
|
-
# source_config = connector_test.get_source_config()
|
34
|
-
|
35
|
-
manifest_dict = yaml.safe_load(self.manifest_path.read_text())
|
36
|
-
if self.components_py_path and self.components_py_path.exists():
|
37
|
-
os.environ["AIRBYTE_ENABLE_UNSAFE_CODE"] = "true"
|
38
|
-
config["__injected_components_py"] = self.components_py_path.read_text()
|
39
|
-
config["__injected_components_py_checksums"] = {
|
40
|
-
"md5": md5_checksum(self.components_py_path),
|
41
|
-
}
|
42
|
-
|
43
|
-
return ConcurrentDeclarativeSource(
|
44
|
-
config=config,
|
45
|
-
catalog=None,
|
46
|
-
state=None,
|
47
|
-
source_config=manifest_dict,
|
48
|
-
)
|
@@ -1,12 +0,0 @@
|
|
1
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
-
"""Base class for destination test suites."""
|
3
|
-
|
4
|
-
from airbyte_cdk.test.declarative.test_suites.connector_base import ConnectorTestSuiteBase
|
5
|
-
|
6
|
-
|
7
|
-
class DestinationTestSuiteBase(ConnectorTestSuiteBase):
|
8
|
-
"""Base class for destination test suites.
|
9
|
-
|
10
|
-
This class provides a base set of functionality for testing destination connectors, and it
|
11
|
-
inherits all generic connector tests from the `ConnectorTestSuiteBase` class.
|
12
|
-
"""
|