airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (518) hide show
  1. airbyte_cdk/__init__.py +355 -6
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +29 -10
  7. airbyte_cdk/connector.py +24 -24
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
  10. airbyte_cdk/connector_builder/main.py +45 -13
  11. airbyte_cdk/connector_builder/message_grouper.py +189 -50
  12. airbyte_cdk/connector_builder/models.py +3 -2
  13. airbyte_cdk/destinations/__init__.py +4 -3
  14. airbyte_cdk/destinations/destination.py +54 -20
  15. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  16. airbyte_cdk/destinations/vector_db_based/config.py +40 -17
  17. airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
  18. airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
  19. airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
  20. airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
  21. airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
  22. airbyte_cdk/entrypoint.py +153 -44
  23. airbyte_cdk/exception_handler.py +21 -3
  24. airbyte_cdk/logger.py +30 -44
  25. airbyte_cdk/models/__init__.py +13 -2
  26. airbyte_cdk/models/airbyte_protocol.py +86 -1
  27. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  28. airbyte_cdk/models/file_transfer_record_message.py +13 -0
  29. airbyte_cdk/models/well_known_types.py +1 -1
  30. airbyte_cdk/sources/__init__.py +5 -1
  31. airbyte_cdk/sources/abstract_source.py +125 -79
  32. airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
  33. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
  34. airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
  35. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
  36. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  37. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
  38. airbyte_cdk/sources/config.py +3 -2
  39. airbyte_cdk/sources/connector_state_manager.py +49 -83
  40. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  41. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
  42. airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
  43. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  44. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  45. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  46. airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
  47. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
  48. airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
  49. airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
  50. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
  51. airbyte_cdk/sources/declarative/auth/token.py +28 -10
  52. airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
  53. airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
  54. airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
  55. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  56. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  57. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
  58. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
  59. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
  60. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1185 -85
  61. airbyte_cdk/sources/declarative/declarative_source.py +5 -2
  62. airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
  63. airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
  64. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
  65. airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
  66. airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
  67. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  68. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  69. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  70. airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
  71. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
  72. airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
  73. airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
  74. airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
  75. airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
  76. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
  77. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  78. airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
  79. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +346 -0
  80. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
  81. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  82. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
  83. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +173 -74
  84. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  85. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  86. airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
  87. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
  88. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
  89. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
  90. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
  91. airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
  92. airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
  93. airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
  94. airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
  95. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  96. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  97. airbyte_cdk/sources/declarative/models/__init__.py +1 -1
  98. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1319 -603
  99. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
  100. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
  101. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
  102. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1759 -225
  103. airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
  104. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  105. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  106. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
  107. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  108. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
  109. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
  110. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
  111. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
  112. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
  113. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
  114. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
  115. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
  116. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
  117. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
  118. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
  119. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
  120. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  121. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
  122. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
  123. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
  124. airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
  125. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
  126. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
  127. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
  128. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
  129. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
  130. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
  131. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
  132. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
  133. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
  134. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
  135. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
  136. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
  137. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  138. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
  139. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
  140. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
  141. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
  142. airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
  143. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  144. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  145. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  146. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  147. airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
  148. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
  149. airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
  150. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
  151. airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
  152. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
  153. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
  154. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
  155. airbyte_cdk/sources/declarative/spec/spec.py +12 -5
  156. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
  157. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
  158. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
  159. airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
  160. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  161. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  162. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  163. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  164. airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
  165. airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
  166. airbyte_cdk/sources/declarative/types.py +19 -110
  167. airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
  168. airbyte_cdk/sources/embedded/base_integration.py +16 -5
  169. airbyte_cdk/sources/embedded/catalog.py +16 -4
  170. airbyte_cdk/sources/embedded/runner.py +19 -3
  171. airbyte_cdk/sources/embedded/tools.py +5 -2
  172. airbyte_cdk/sources/file_based/README.md +152 -0
  173. airbyte_cdk/sources/file_based/__init__.py +24 -0
  174. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
  175. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
  176. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
  177. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
  178. airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
  179. airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
  180. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  181. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
  182. airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
  183. airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
  184. airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
  185. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
  186. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
  187. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
  188. airbyte_cdk/sources/file_based/exceptions.py +18 -15
  189. airbyte_cdk/sources/file_based/file_based_source.py +140 -33
  190. airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
  191. airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
  192. airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
  193. airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
  194. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  195. airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
  196. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
  197. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
  198. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
  199. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
  200. airbyte_cdk/sources/file_based/remote_file.py +1 -1
  201. airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
  202. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
  203. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
  204. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
  205. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
  206. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
  207. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
  208. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
  209. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
  210. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
  211. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
  212. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
  213. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
  214. airbyte_cdk/sources/http_logger.py +8 -3
  215. airbyte_cdk/sources/message/__init__.py +7 -1
  216. airbyte_cdk/sources/message/repository.py +18 -4
  217. airbyte_cdk/sources/source.py +42 -38
  218. airbyte_cdk/sources/streams/__init__.py +2 -2
  219. airbyte_cdk/sources/streams/availability_strategy.py +54 -3
  220. airbyte_cdk/sources/streams/call_rate.py +64 -21
  221. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  222. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  223. airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
  224. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  225. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  226. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  227. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  228. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
  229. airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
  230. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
  231. airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
  232. airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
  233. airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
  234. airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
  235. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
  236. airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
  237. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
  238. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  239. airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
  240. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
  241. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
  242. airbyte_cdk/sources/streams/core.py +412 -87
  243. airbyte_cdk/sources/streams/http/__init__.py +2 -1
  244. airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
  245. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  246. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  247. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  248. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  249. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  250. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  251. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  252. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  253. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  254. airbyte_cdk/sources/streams/http/exceptions.py +27 -7
  255. airbyte_cdk/sources/streams/http/http.py +369 -246
  256. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  257. airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
  258. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
  259. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
  260. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
  261. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
  262. airbyte_cdk/sources/types.py +154 -0
  263. airbyte_cdk/sources/utils/record_helper.py +36 -21
  264. airbyte_cdk/sources/utils/schema_helpers.py +13 -6
  265. airbyte_cdk/sources/utils/slice_logger.py +4 -1
  266. airbyte_cdk/sources/utils/transform.py +54 -20
  267. airbyte_cdk/sql/_util/hashing.py +34 -0
  268. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  269. airbyte_cdk/sql/constants.py +32 -0
  270. airbyte_cdk/sql/exceptions.py +235 -0
  271. airbyte_cdk/sql/secrets.py +123 -0
  272. airbyte_cdk/sql/shared/__init__.py +15 -0
  273. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  274. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  275. airbyte_cdk/sql/types.py +160 -0
  276. airbyte_cdk/test/catalog_builder.py +70 -18
  277. airbyte_cdk/test/entrypoint_wrapper.py +117 -42
  278. airbyte_cdk/test/mock_http/__init__.py +1 -1
  279. airbyte_cdk/test/mock_http/matcher.py +6 -0
  280. airbyte_cdk/test/mock_http/mocker.py +57 -10
  281. airbyte_cdk/test/mock_http/request.py +19 -3
  282. airbyte_cdk/test/mock_http/response.py +3 -1
  283. airbyte_cdk/test/mock_http/response_builder.py +32 -16
  284. airbyte_cdk/test/state_builder.py +18 -10
  285. airbyte_cdk/test/utils/__init__.py +1 -0
  286. airbyte_cdk/test/utils/data.py +24 -0
  287. airbyte_cdk/test/utils/http_mocking.py +16 -0
  288. airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
  289. airbyte_cdk/test/utils/reading.py +26 -0
  290. airbyte_cdk/utils/__init__.py +2 -1
  291. airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
  292. airbyte_cdk/utils/analytics_message.py +10 -2
  293. airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
  294. airbyte_cdk/utils/event_timing.py +10 -10
  295. airbyte_cdk/utils/mapping_helpers.py +3 -1
  296. airbyte_cdk/utils/message_utils.py +20 -11
  297. airbyte_cdk/utils/print_buffer.py +75 -0
  298. airbyte_cdk/utils/schema_inferrer.py +198 -28
  299. airbyte_cdk/utils/slice_hasher.py +30 -0
  300. airbyte_cdk/utils/spec_schema_transformations.py +6 -3
  301. airbyte_cdk/utils/stream_status_utils.py +8 -1
  302. airbyte_cdk/utils/traced_exception.py +61 -21
  303. airbyte_cdk-6.17.1.dev0.dist-info/METADATA +109 -0
  304. airbyte_cdk-6.17.1.dev0.dist-info/RECORD +350 -0
  305. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/WHEEL +1 -2
  306. airbyte_cdk-6.17.1.dev0.dist-info/entry_points.txt +3 -0
  307. airbyte_cdk/sources/declarative/create_partial.py +0 -92
  308. airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
  309. airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
  310. airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
  311. airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
  312. airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
  313. airbyte_cdk/sources/deprecated/base_source.py +0 -94
  314. airbyte_cdk/sources/deprecated/client.py +0 -99
  315. airbyte_cdk/sources/singer/__init__.py +0 -8
  316. airbyte_cdk/sources/singer/singer_helpers.py +0 -304
  317. airbyte_cdk/sources/singer/source.py +0 -186
  318. airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
  319. airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
  320. airbyte_cdk/sources/streams/http/auth/core.py +0 -29
  321. airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
  322. airbyte_cdk/sources/streams/http/auth/token.py +0 -47
  323. airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
  324. airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
  325. airbyte_cdk/sources/utils/schema_models.py +0 -84
  326. airbyte_cdk-0.72.1.dist-info/METADATA +0 -243
  327. airbyte_cdk-0.72.1.dist-info/RECORD +0 -466
  328. airbyte_cdk-0.72.1.dist-info/top_level.txt +0 -3
  329. source_declarative_manifest/main.py +0 -29
  330. unit_tests/connector_builder/__init__.py +0 -3
  331. unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
  332. unit_tests/connector_builder/test_message_grouper.py +0 -713
  333. unit_tests/connector_builder/utils.py +0 -27
  334. unit_tests/destinations/test_destination.py +0 -243
  335. unit_tests/singer/test_singer_helpers.py +0 -56
  336. unit_tests/singer/test_singer_source.py +0 -112
  337. unit_tests/sources/__init__.py +0 -0
  338. unit_tests/sources/concurrent_source/__init__.py +0 -3
  339. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
  340. unit_tests/sources/declarative/__init__.py +0 -3
  341. unit_tests/sources/declarative/auth/__init__.py +0 -3
  342. unit_tests/sources/declarative/auth/test_oauth.py +0 -331
  343. unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
  344. unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
  345. unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
  346. unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
  347. unit_tests/sources/declarative/checks/__init__.py +0 -3
  348. unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
  349. unit_tests/sources/declarative/decoders/__init__.py +0 -0
  350. unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
  351. unit_tests/sources/declarative/external_component.py +0 -13
  352. unit_tests/sources/declarative/extractors/__init__.py +0 -3
  353. unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
  354. unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
  355. unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
  356. unit_tests/sources/declarative/incremental/__init__.py +0 -0
  357. unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
  358. unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
  359. unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
  360. unit_tests/sources/declarative/interpolation/__init__.py +0 -3
  361. unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
  362. unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
  363. unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
  364. unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
  365. unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
  366. unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
  367. unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
  368. unit_tests/sources/declarative/parsers/__init__.py +0 -3
  369. unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
  370. unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
  371. unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1847
  372. unit_tests/sources/declarative/parsers/testing_components.py +0 -36
  373. unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
  374. unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
  375. unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
  376. unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
  377. unit_tests/sources/declarative/requesters/__init__.py +0 -3
  378. unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
  379. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
  380. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
  381. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
  382. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
  383. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
  384. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
  385. unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
  386. unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
  387. unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
  388. unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
  389. unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
  390. unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
  391. unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
  392. unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
  393. unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
  394. unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
  395. unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
  396. unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
  397. unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
  398. unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
  399. unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
  400. unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
  401. unit_tests/sources/declarative/retrievers/__init__.py +0 -3
  402. unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
  403. unit_tests/sources/declarative/schema/__init__.py +0 -6
  404. unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
  405. unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
  406. unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
  407. unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
  408. unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
  409. unit_tests/sources/declarative/states/__init__.py +0 -3
  410. unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
  411. unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
  412. unit_tests/sources/declarative/test_create_partial.py +0 -83
  413. unit_tests/sources/declarative/test_declarative_stream.py +0 -103
  414. unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
  415. unit_tests/sources/declarative/test_types.py +0 -39
  416. unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
  417. unit_tests/sources/file_based/__init__.py +0 -0
  418. unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
  419. unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
  420. unit_tests/sources/file_based/config/__init__.py +0 -0
  421. unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
  422. unit_tests/sources/file_based/config/test_csv_format.py +0 -34
  423. unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
  424. unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
  425. unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
  426. unit_tests/sources/file_based/file_types/__init__.py +0 -0
  427. unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
  428. unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
  429. unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
  430. unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
  431. unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
  432. unit_tests/sources/file_based/helpers.py +0 -70
  433. unit_tests/sources/file_based/in_memory_files_source.py +0 -211
  434. unit_tests/sources/file_based/scenarios/__init__.py +0 -0
  435. unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
  436. unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
  437. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
  438. unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
  439. unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
  440. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
  441. unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
  442. unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
  443. unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
  444. unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
  445. unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
  446. unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
  447. unit_tests/sources/file_based/stream/__init__.py +0 -0
  448. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  449. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
  450. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
  451. unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
  452. unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
  453. unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
  454. unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
  455. unit_tests/sources/file_based/test_scenarios.py +0 -253
  456. unit_tests/sources/file_based/test_schema_helpers.py +0 -346
  457. unit_tests/sources/fixtures/__init__.py +0 -3
  458. unit_tests/sources/fixtures/source_test_fixture.py +0 -153
  459. unit_tests/sources/message/__init__.py +0 -0
  460. unit_tests/sources/message/test_repository.py +0 -153
  461. unit_tests/sources/streams/__init__.py +0 -0
  462. unit_tests/sources/streams/concurrent/__init__.py +0 -3
  463. unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
  464. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
  465. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
  466. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
  467. unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
  468. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
  469. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
  470. unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
  471. unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
  472. unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
  473. unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
  474. unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
  475. unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
  476. unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
  477. unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
  478. unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
  479. unit_tests/sources/streams/http/__init__.py +0 -0
  480. unit_tests/sources/streams/http/auth/__init__.py +0 -0
  481. unit_tests/sources/streams/http/auth/test_auth.py +0 -173
  482. unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
  483. unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
  484. unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
  485. unit_tests/sources/streams/http/test_http.py +0 -635
  486. unit_tests/sources/streams/test_availability_strategy.py +0 -70
  487. unit_tests/sources/streams/test_call_rate.py +0 -300
  488. unit_tests/sources/streams/test_stream_read.py +0 -405
  489. unit_tests/sources/streams/test_streams_core.py +0 -184
  490. unit_tests/sources/test_abstract_source.py +0 -1442
  491. unit_tests/sources/test_concurrent_source.py +0 -112
  492. unit_tests/sources/test_config.py +0 -92
  493. unit_tests/sources/test_connector_state_manager.py +0 -482
  494. unit_tests/sources/test_http_logger.py +0 -252
  495. unit_tests/sources/test_integration_source.py +0 -86
  496. unit_tests/sources/test_source.py +0 -684
  497. unit_tests/sources/test_source_read.py +0 -460
  498. unit_tests/test/__init__.py +0 -0
  499. unit_tests/test/mock_http/__init__.py +0 -0
  500. unit_tests/test/mock_http/test_matcher.py +0 -53
  501. unit_tests/test/mock_http/test_mocker.py +0 -214
  502. unit_tests/test/mock_http/test_request.py +0 -117
  503. unit_tests/test/mock_http/test_response_builder.py +0 -177
  504. unit_tests/test/test_entrypoint_wrapper.py +0 -240
  505. unit_tests/utils/__init__.py +0 -0
  506. unit_tests/utils/test_datetime_format_inferrer.py +0 -60
  507. unit_tests/utils/test_mapping_helpers.py +0 -54
  508. unit_tests/utils/test_message_utils.py +0 -91
  509. unit_tests/utils/test_rate_limiting.py +0 -26
  510. unit_tests/utils/test_schema_inferrer.py +0 -202
  511. unit_tests/utils/test_secret_utils.py +0 -135
  512. unit_tests/utils/test_stream_status_utils.py +0 -61
  513. unit_tests/utils/test_traced_exception.py +0 -107
  514. /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
  515. {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
  516. {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
  517. {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
  518. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/LICENSE.txt +0 -0
@@ -1,36 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- from dataclasses import dataclass
6
- from typing import List, Optional
7
-
8
- from airbyte_cdk.sources.declarative.extractors import DpathExtractor
9
- from airbyte_cdk.sources.declarative.partition_routers import SubstreamPartitionRouter
10
- from airbyte_cdk.sources.declarative.requesters import RequestOption
11
- from airbyte_cdk.sources.declarative.requesters.error_handlers import DefaultErrorHandler
12
- from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy
13
-
14
-
15
- @dataclass
16
- class TestingSomeComponent(DefaultErrorHandler):
17
- """
18
- A basic test class with various field permutations used to test manifests with custom components
19
- """
20
-
21
- subcomponent_field_with_hint: DpathExtractor = DpathExtractor(field_path=[], config={}, parameters={})
22
- basic_field: str = ""
23
- optional_subcomponent_field: Optional[RequestOption] = None
24
- list_of_subcomponents: List[RequestOption] = None
25
- without_hint = None
26
- paginator: DefaultPaginator = None
27
-
28
-
29
- @dataclass
30
- class TestingCustomSubstreamPartitionRouter(SubstreamPartitionRouter):
31
- """
32
- A test class based on a SubstreamPartitionRouter used for testing manifests that use custom components.
33
- """
34
-
35
- custom_field: str
36
- custom_pagination_strategy: PaginationStrategy
@@ -1,3 +0,0 @@
1
- #
2
- # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
3
- #
@@ -1,155 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- import pytest as pytest
6
- from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import ListPartitionRouter
7
- from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType
8
- from airbyte_cdk.sources.declarative.types import StreamSlice
9
-
10
- partition_values = ["customer", "store", "subscription"]
11
- cursor_field = "owner_resource"
12
- parameters = {"cursor_field": "owner_resource"}
13
-
14
-
15
- @pytest.mark.parametrize(
16
- "partition_values, cursor_field, expected_slices",
17
- [
18
- (
19
- ["customer", "store", "subscription"],
20
- "owner_resource",
21
- [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}),
22
- StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}),
23
- StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})],
24
- ),
25
- (
26
- '["customer", "store", "subscription"]',
27
- "owner_resource",
28
- [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}),
29
- StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}),
30
- StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})],
31
- ),
32
- (
33
- '["customer", "store", "subscription"]',
34
- "{{ parameters['cursor_field'] }}",
35
- [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}),
36
- StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}),
37
- StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})],
38
- ),
39
- ],
40
- ids=[
41
- "test_single_element",
42
- "test_input_list_is_string",
43
- "test_using_cursor_from_parameters",
44
- ],
45
- )
46
- def test_list_partition_router(partition_values, cursor_field, expected_slices):
47
- slicer = ListPartitionRouter(values=partition_values, cursor_field=cursor_field, config={}, parameters=parameters)
48
- slices = [s for s in slicer.stream_slices()]
49
- assert slices == expected_slices
50
- assert all(isinstance(s, StreamSlice) for s in slices)
51
-
52
-
53
- @pytest.mark.parametrize(
54
- "request_option, expected_req_params, expected_headers, expected_body_json, expected_body_data",
55
- [
56
- (
57
- RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="owner_resource"),
58
- {"owner_resource": "customer"},
59
- {},
60
- {},
61
- {},
62
- ),
63
- (
64
- RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="owner_resource"),
65
- {},
66
- {"owner_resource": "customer"},
67
- {},
68
- {},
69
- ),
70
- (
71
- RequestOption(inject_into=RequestOptionType.body_json, parameters={}, field_name="owner_resource"),
72
- {},
73
- {},
74
- {"owner_resource": "customer"},
75
- {},
76
- ),
77
- (
78
- RequestOption(inject_into=RequestOptionType.body_data, parameters={}, field_name="owner_resource"),
79
- {},
80
- {},
81
- {},
82
- {"owner_resource": "customer"},
83
- ),
84
- ],
85
- ids=[
86
- "test_inject_into_req_param",
87
- "test_pass_by_header",
88
- "test_inject_into_body_json",
89
- "test_inject_into_body_data",
90
- ],
91
- )
92
- def test_request_option(request_option, expected_req_params, expected_headers, expected_body_json, expected_body_data):
93
- partition_router = ListPartitionRouter(
94
- values=partition_values, cursor_field=cursor_field, config={}, request_option=request_option, parameters={}
95
- )
96
- stream_slice = {cursor_field: "customer"}
97
-
98
- assert expected_req_params == partition_router.get_request_params(stream_slice=stream_slice)
99
- assert expected_headers == partition_router.get_request_headers(stream_slice=stream_slice)
100
- assert expected_body_json == partition_router.get_request_body_json(stream_slice=stream_slice)
101
- assert expected_body_data == partition_router.get_request_body_data(stream_slice=stream_slice)
102
-
103
-
104
- @pytest.mark.parametrize(
105
- "stream_slice",
106
- [
107
- pytest.param({}, id="test_request_option_is_empty_if_empty_stream_slice"),
108
- pytest.param({"not the cursor": "value"}, id="test_request_option_is_empty_if_the_stream_slice_does_not_have_cursor_field"),
109
- pytest.param(None, id="test_request_option_is_empty_if_no_stream_slice")
110
- ]
111
- )
112
- def test_request_option_is_empty_if_no_stream_slice(stream_slice):
113
- request_option = RequestOption(inject_into=RequestOptionType.body_data, parameters={}, field_name="owner_resource")
114
- partition_router = ListPartitionRouter(
115
- values=partition_values, cursor_field=cursor_field, config={}, request_option=request_option, parameters={}
116
- )
117
- assert {} == partition_router.get_request_body_data(stream_slice=stream_slice)
118
-
119
-
120
- @pytest.mark.parametrize(
121
- "field_name_interpolation, expected_request_params",
122
- [
123
- ("{{parameters['partition_name']}}", {"parameters_partition": "customer"}),
124
- ("{{config['partition_name']}}", {"config_partition": "customer"}),
125
- ],
126
- ids=[
127
- "parameters_interpolation",
128
- "config_interpolation",
129
- ],
130
- )
131
- def test_request_options_interpolation(field_name_interpolation: str, expected_request_params: dict):
132
- config = {"partition_name": "config_partition"}
133
- parameters = {"partition_name": "parameters_partition"}
134
- request_option = RequestOption(
135
- inject_into=RequestOptionType.request_parameter, parameters=parameters, field_name=field_name_interpolation
136
- )
137
- partition_router = ListPartitionRouter(
138
- values=partition_values, cursor_field=cursor_field, config=config, request_option=request_option, parameters=parameters
139
- )
140
- stream_slice = {cursor_field: "customer"}
141
-
142
- assert expected_request_params == partition_router.get_request_params(stream_slice=stream_slice)
143
-
144
-
145
- def test_request_option_before_updating_cursor():
146
- request_option = RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="owner_resource")
147
- partition_router = ListPartitionRouter(
148
- values=partition_values, cursor_field=cursor_field, config={}, request_option=request_option, parameters={}
149
- )
150
- stream_slice = {cursor_field: "customer"}
151
-
152
- assert {} == partition_router.get_request_params(stream_slice)
153
- assert {} == partition_router.get_request_headers()
154
- assert {} == partition_router.get_request_body_json()
155
- assert {} == partition_router.get_request_body_data()
@@ -1,14 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import SinglePartitionRouter
6
- from airbyte_cdk.sources.declarative.types import StreamSlice
7
-
8
-
9
- def test():
10
- iterator = SinglePartitionRouter(parameters={})
11
-
12
- stream_slices = iterator.stream_slices()
13
- next_slice = next(stream_slices)
14
- assert next_slice == StreamSlice(partition={}, cursor_slice={})
@@ -1,404 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- from typing import Any, Iterable, List, Mapping, Optional, Union
6
-
7
- import pytest as pytest
8
- from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, SyncMode, Type
9
- from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
10
- from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import StreamSlice
11
- from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig, SubstreamPartitionRouter
12
- from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType
13
- from airbyte_cdk.sources.declarative.types import Record
14
-
15
- parent_records = [{"id": 1, "data": "data1"}, {"id": 2, "data": "data2"}]
16
- more_records = [{"id": 10, "data": "data10", "slice": "second_parent"}, {"id": 20, "data": "data20", "slice": "second_parent"}]
17
-
18
- data_first_parent_slice = [{"id": 0, "slice": "first", "data": "A"}, {"id": 1, "slice": "first", "data": "B"}]
19
- data_second_parent_slice = [{"id": 2, "slice": "second", "data": "C"}]
20
- data_third_parent_slice = []
21
- all_parent_data = data_first_parent_slice + data_second_parent_slice + data_third_parent_slice
22
- parent_slices = [{"slice": "first"}, {"slice": "second"}, {"slice": "third"}]
23
- second_parent_stream_slice = [StreamSlice(partition={"slice": "second_parent"}, cursor_slice={})]
24
-
25
-
26
- class MockStream(DeclarativeStream):
27
- def __init__(self, slices, records, name):
28
- self._slices = slices
29
- self._records = records
30
- self._name = name
31
-
32
- @property
33
- def name(self) -> str:
34
- return self._name
35
-
36
- @property
37
- def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]:
38
- return "id"
39
-
40
- def stream_slices(
41
- self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
42
- ) -> Iterable[Optional[StreamSlice]]:
43
- for s in self._slices:
44
- if isinstance(s, StreamSlice):
45
- yield s
46
- else:
47
- yield StreamSlice(partition=s, cursor_slice={})
48
-
49
- def read_records(
50
- self,
51
- sync_mode: SyncMode,
52
- cursor_field: List[str] = None,
53
- stream_slice: Mapping[str, Any] = None,
54
- stream_state: Mapping[str, Any] = None,
55
- ) -> Iterable[Mapping[str, Any]]:
56
- # The parent stream's records should always be read as full refresh
57
- assert sync_mode == SyncMode.full_refresh
58
- if not stream_slice:
59
- yield from self._records
60
- else:
61
- yield from [r for r in self._records if r["slice"] == stream_slice["slice"]]
62
-
63
-
64
- @pytest.mark.parametrize(
65
- "parent_stream_configs, expected_slices",
66
- [
67
- ([], None),
68
- (
69
- [
70
- ParentStreamConfig(
71
- stream=MockStream([{}], [], "first_stream"),
72
- parent_key="id",
73
- partition_field="first_stream_id",
74
- parameters={},
75
- config={},
76
- )
77
- ],
78
- [],
79
- ),
80
- (
81
- [
82
- ParentStreamConfig(
83
- stream=MockStream([{}], parent_records, "first_stream"),
84
- parent_key="id",
85
- partition_field="first_stream_id",
86
- parameters={},
87
- config={},
88
- )
89
- ],
90
- [{"first_stream_id": 1, "parent_slice": {}}, {"first_stream_id": 2, "parent_slice": {}}],
91
- ),
92
- (
93
- [
94
- ParentStreamConfig(
95
- stream=MockStream(parent_slices, all_parent_data, "first_stream"),
96
- parent_key="id",
97
- partition_field="first_stream_id",
98
- parameters={},
99
- config={},
100
- )
101
- ],
102
- [
103
- {"parent_slice": {"slice": "first"}, "first_stream_id": 0},
104
- {"parent_slice": {"slice": "first"}, "first_stream_id": 1},
105
- {"parent_slice": {"slice": "second"}, "first_stream_id": 2},
106
- ],
107
- ),
108
- (
109
- [
110
- ParentStreamConfig(
111
- stream=MockStream([StreamSlice(partition=p, cursor_slice={"start": 0, "end": 1}) for p in parent_slices], all_parent_data, "first_stream"),
112
- parent_key="id",
113
- partition_field="first_stream_id",
114
- parameters={},
115
- config={},
116
- )
117
- ],
118
- [
119
- {"parent_slice": {"slice": "first"}, "first_stream_id": 0},
120
- {"parent_slice": {"slice": "first"}, "first_stream_id": 1},
121
- {"parent_slice": {"slice": "second"}, "first_stream_id": 2},
122
- ],
123
- ),
124
- (
125
- [
126
- ParentStreamConfig(
127
- stream=MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"),
128
- parent_key="id",
129
- partition_field="first_stream_id",
130
- parameters={},
131
- config={},
132
- ),
133
- ParentStreamConfig(
134
- stream=MockStream(second_parent_stream_slice, more_records, "second_stream"),
135
- parent_key="id",
136
- partition_field="second_stream_id",
137
- parameters={},
138
- config={},
139
- ),
140
- ],
141
- [
142
- {"parent_slice": {"slice": "first"}, "first_stream_id": 0},
143
- {"parent_slice": {"slice": "first"}, "first_stream_id": 1},
144
- {"parent_slice": {"slice": "second"}, "first_stream_id": 2},
145
- {"parent_slice": {"slice": "second_parent"}, "second_stream_id": 10},
146
- {"parent_slice": {"slice": "second_parent"}, "second_stream_id": 20},
147
- ],
148
- ),
149
- (
150
- [
151
- ParentStreamConfig(
152
- stream=MockStream([{}], [{"id": 0}, {"id": 1}, {"_id": 2}, {"id": 3}], "first_stream"),
153
- parent_key="id",
154
- partition_field="first_stream_id",
155
- parameters={},
156
- config={},
157
- )
158
- ],
159
- [
160
- {"first_stream_id": 0, "parent_slice": {}},
161
- {"first_stream_id": 1, "parent_slice": {}},
162
- {"first_stream_id": 3, "parent_slice": {}},
163
- ],
164
- ),
165
- (
166
- [
167
- ParentStreamConfig(
168
- stream=MockStream([{}], [{"a": {"b": 0}}, {"a": {"b": 1}}, {"a": {"c": 2}}, {"a": {"b": 3}}], "first_stream"),
169
- parent_key="a/b",
170
- partition_field="first_stream_id",
171
- parameters={},
172
- config={},
173
- )
174
- ],
175
- [
176
- {"first_stream_id": 0, "parent_slice": {}},
177
- {"first_stream_id": 1, "parent_slice": {}},
178
- {"first_stream_id": 3, "parent_slice": {}},
179
- ],
180
- ),
181
- ],
182
- ids=[
183
- "test_no_parents",
184
- "test_single_parent_slices_no_records",
185
- "test_single_parent_slices_with_records",
186
- "test_with_parent_slices_and_records",
187
- "test_multiple_parent_streams",
188
- "test_cursor_values_are_removed_from_parent_slices",
189
- "test_missed_parent_key",
190
- "test_dpath_extraction",
191
- ],
192
- )
193
- def test_substream_slicer(parent_stream_configs, expected_slices):
194
- if expected_slices is None:
195
- try:
196
- SubstreamPartitionRouter(parent_stream_configs=parent_stream_configs, parameters={}, config={})
197
- assert False
198
- except ValueError:
199
- return
200
- partition_router = SubstreamPartitionRouter(parent_stream_configs=parent_stream_configs, parameters={}, config={})
201
- slices = [s for s in partition_router.stream_slices()]
202
- assert slices == expected_slices
203
-
204
-
205
- @pytest.mark.parametrize(
206
- "parent_stream_request_parameters, expected_req_params, expected_headers, expected_body_json, expected_body_data",
207
- [
208
- (
209
- [
210
- RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="first_stream"),
211
- RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="second_stream"),
212
- ],
213
- {"first_stream": "1234", "second_stream": "4567"},
214
- {},
215
- {},
216
- {},
217
- ),
218
- (
219
- [
220
- RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="first_stream"),
221
- RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="second_stream"),
222
- ],
223
- {},
224
- {"first_stream": "1234", "second_stream": "4567"},
225
- {},
226
- {},
227
- ),
228
- (
229
- [
230
- RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="first_stream"),
231
- RequestOption(inject_into=RequestOptionType.header, parameters={}, field_name="second_stream"),
232
- ],
233
- {"first_stream": "1234"},
234
- {"second_stream": "4567"},
235
- {},
236
- {},
237
- ),
238
- (
239
- [
240
- RequestOption(inject_into=RequestOptionType.body_json, parameters={}, field_name="first_stream"),
241
- RequestOption(inject_into=RequestOptionType.body_json, parameters={}, field_name="second_stream"),
242
- ],
243
- {},
244
- {},
245
- {"first_stream": "1234", "second_stream": "4567"},
246
- {},
247
- ),
248
- (
249
- [
250
- RequestOption(inject_into=RequestOptionType.body_data, parameters={}, field_name="first_stream"),
251
- RequestOption(inject_into=RequestOptionType.body_data, parameters={}, field_name="second_stream"),
252
- ],
253
- {},
254
- {},
255
- {},
256
- {"first_stream": "1234", "second_stream": "4567"},
257
- ),
258
- ],
259
- ids=[
260
- "test_request_option_in_request_param",
261
- "test_request_option_in_header",
262
- "test_request_option_in_param_and_header",
263
- "test_request_option_in_body_json",
264
- "test_request_option_in_body_data",
265
- ],
266
- )
267
- def test_request_option(
268
- parent_stream_request_parameters,
269
- expected_req_params,
270
- expected_headers,
271
- expected_body_json,
272
- expected_body_data,
273
- ):
274
- partition_router = SubstreamPartitionRouter(
275
- parent_stream_configs=[
276
- ParentStreamConfig(
277
- stream=MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"),
278
- parent_key="id",
279
- partition_field="first_stream_id",
280
- parameters={},
281
- config={},
282
- request_option=parent_stream_request_parameters[0],
283
- ),
284
- ParentStreamConfig(
285
- stream=MockStream(second_parent_stream_slice, more_records, "second_stream"),
286
- parent_key="id",
287
- partition_field="second_stream_id",
288
- parameters={},
289
- config={},
290
- request_option=parent_stream_request_parameters[1],
291
- ),
292
- ],
293
- parameters={},
294
- config={},
295
- )
296
- stream_slice = {"first_stream_id": "1234", "second_stream_id": "4567"}
297
-
298
- assert expected_req_params == partition_router.get_request_params(stream_slice=stream_slice)
299
- assert expected_headers == partition_router.get_request_headers(stream_slice=stream_slice)
300
- assert expected_body_json == partition_router.get_request_body_json(stream_slice=stream_slice)
301
- assert expected_body_data == partition_router.get_request_body_data(stream_slice=stream_slice)
302
-
303
-
304
- @pytest.mark.parametrize(
305
- "field_name_first_stream, field_name_second_stream, expected_request_params",
306
- [
307
- (
308
- "{{parameters['field_name_first_stream']}}",
309
- "{{parameters['field_name_second_stream']}}",
310
- {"parameter_first_stream_id": "1234", "parameter_second_stream_id": "4567"},
311
- ),
312
- (
313
- "{{config['field_name_first_stream']}}",
314
- "{{config['field_name_second_stream']}}",
315
- {"config_first_stream_id": "1234", "config_second_stream_id": "4567"},
316
- ),
317
- ],
318
- ids=[
319
- "parameters_interpolation",
320
- "config_interpolation",
321
- ],
322
- )
323
- def test_request_params_interpolation_for_parent_stream(
324
- field_name_first_stream: str, field_name_second_stream: str, expected_request_params: dict
325
- ):
326
- config = {"field_name_first_stream": "config_first_stream_id", "field_name_second_stream": "config_second_stream_id"}
327
- parameters = {"field_name_first_stream": "parameter_first_stream_id", "field_name_second_stream": "parameter_second_stream_id"}
328
- partition_router = SubstreamPartitionRouter(
329
- parent_stream_configs=[
330
- ParentStreamConfig(
331
- stream=MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"),
332
- parent_key="id",
333
- partition_field="first_stream_id",
334
- parameters=parameters,
335
- config=config,
336
- request_option=RequestOption(
337
- inject_into=RequestOptionType.request_parameter, parameters=parameters, field_name=field_name_first_stream
338
- ),
339
- ),
340
- ParentStreamConfig(
341
- stream=MockStream(second_parent_stream_slice, more_records, "second_stream"),
342
- parent_key="id",
343
- partition_field="second_stream_id",
344
- parameters=parameters,
345
- config=config,
346
- request_option=RequestOption(
347
- inject_into=RequestOptionType.request_parameter, parameters=parameters, field_name=field_name_second_stream
348
- ),
349
- ),
350
- ],
351
- parameters=parameters,
352
- config=config,
353
- )
354
- stream_slice = {"first_stream_id": "1234", "second_stream_id": "4567"}
355
-
356
- assert expected_request_params == partition_router.get_request_params(stream_slice=stream_slice)
357
-
358
-
359
- def test_given_record_is_airbyte_message_when_stream_slices_then_use_record_data():
360
- parent_slice = {}
361
- partition_router = SubstreamPartitionRouter(
362
- parent_stream_configs=[
363
- ParentStreamConfig(
364
- stream=MockStream(
365
- [parent_slice],
366
- [
367
- AirbyteMessage(
368
- type=Type.RECORD, record=AirbyteRecordMessage(data={"id": "record value"}, emitted_at=0, stream="stream")
369
- )
370
- ],
371
- "first_stream",
372
- ),
373
- parent_key="id",
374
- partition_field="partition_field",
375
- parameters={},
376
- config={},
377
- )
378
- ],
379
- parameters={},
380
- config={},
381
- )
382
-
383
- slices = list(partition_router.stream_slices())
384
- assert slices == [{"partition_field": "record value", "parent_slice": parent_slice}]
385
-
386
-
387
- def test_given_record_is_record_object_when_stream_slices_then_use_record_data():
388
- parent_slice = {}
389
- partition_router = SubstreamPartitionRouter(
390
- parent_stream_configs=[
391
- ParentStreamConfig(
392
- stream=MockStream([parent_slice], [Record({"id": "record value"}, {})], "first_stream"),
393
- parent_key="id",
394
- partition_field="partition_field",
395
- parameters={},
396
- config={},
397
- )
398
- ],
399
- parameters={},
400
- config={},
401
- )
402
-
403
- slices = list(partition_router.stream_slices())
404
- assert slices == [{"partition_field": "record value", "parent_slice": parent_slice}]
@@ -1,3 +0,0 @@
1
- #
2
- # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3
- #
@@ -1,3 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
@@ -1,3 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
@@ -1,34 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- from unittest.mock import MagicMock
6
-
7
- import pytest
8
- from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.constant_backoff_strategy import ConstantBackoffStrategy
9
-
10
- BACKOFF_TIME = 10
11
- PARAMETERS_BACKOFF_TIME = 20
12
- CONFIG_BACKOFF_TIME = 30
13
-
14
-
15
- @pytest.mark.parametrize(
16
- "test_name, attempt_count, backofftime, expected_backoff_time",
17
- [
18
- ("test_constant_backoff_first_attempt", 1, BACKOFF_TIME, BACKOFF_TIME),
19
- ("test_constant_backoff_first_attempt_float", 1, 6.7, 6.7),
20
- ("test_constant_backoff_attempt_round_float", 1.0, 6.7, 6.7),
21
- ("test_constant_backoff_attempt_round_float", 1.5, 6.7, 6.7),
22
- ("test_constant_backoff_first_attempt_round_float", 1, 10.0, BACKOFF_TIME),
23
- ("test_constant_backoff_second_attempt_round_float", 2, 10.0, BACKOFF_TIME),
24
- ("test_constant_backoff_from_parameters", 1, "{{ parameters['backoff'] }}", PARAMETERS_BACKOFF_TIME),
25
- ("test_constant_backoff_from_config", 1, "{{ config['backoff'] }}", CONFIG_BACKOFF_TIME),
26
- ],
27
- )
28
- def test_constant_backoff(test_name, attempt_count, backofftime, expected_backoff_time):
29
- response_mock = MagicMock()
30
- backoff_strategy = ConstantBackoffStrategy(
31
- parameters={"backoff": PARAMETERS_BACKOFF_TIME}, backoff_time_in_seconds=backofftime, config={"backoff": CONFIG_BACKOFF_TIME}
32
- )
33
- backoff = backoff_strategy.backoff(response_mock, attempt_count)
34
- assert backoff == expected_backoff_time