airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (517) hide show
  1. airbyte_cdk/__init__.py +355 -6
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +29 -10
  7. airbyte_cdk/connector.py +24 -24
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
  10. airbyte_cdk/connector_builder/main.py +45 -13
  11. airbyte_cdk/connector_builder/message_grouper.py +189 -50
  12. airbyte_cdk/connector_builder/models.py +3 -2
  13. airbyte_cdk/destinations/__init__.py +4 -3
  14. airbyte_cdk/destinations/destination.py +54 -20
  15. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  16. airbyte_cdk/destinations/vector_db_based/config.py +40 -17
  17. airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
  18. airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
  19. airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
  20. airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
  21. airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
  22. airbyte_cdk/entrypoint.py +153 -44
  23. airbyte_cdk/exception_handler.py +21 -3
  24. airbyte_cdk/logger.py +30 -44
  25. airbyte_cdk/models/__init__.py +13 -2
  26. airbyte_cdk/models/airbyte_protocol.py +86 -1
  27. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  28. airbyte_cdk/models/file_transfer_record_message.py +13 -0
  29. airbyte_cdk/models/well_known_types.py +1 -1
  30. airbyte_cdk/sources/__init__.py +5 -1
  31. airbyte_cdk/sources/abstract_source.py +125 -79
  32. airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
  33. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
  34. airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
  35. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
  36. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  37. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
  38. airbyte_cdk/sources/config.py +3 -2
  39. airbyte_cdk/sources/connector_state_manager.py +49 -83
  40. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  41. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
  42. airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
  43. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  44. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  45. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  46. airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
  47. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
  48. airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
  49. airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
  50. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
  51. airbyte_cdk/sources/declarative/auth/token.py +28 -10
  52. airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
  53. airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
  54. airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
  55. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  56. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  57. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +421 -0
  58. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
  59. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
  60. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1213 -88
  61. airbyte_cdk/sources/declarative/declarative_source.py +5 -2
  62. airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
  63. airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
  64. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
  65. airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
  66. airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
  67. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  68. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  69. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  70. airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
  71. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
  72. airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
  73. airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
  74. airbyte_cdk/sources/declarative/extractors/record_filter.py +65 -8
  75. airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
  76. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
  77. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  78. airbyte_cdk/sources/declarative/incremental/__init__.py +25 -3
  79. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
  80. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  81. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
  82. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +159 -74
  83. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  84. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  85. airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
  86. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
  87. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
  88. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
  89. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
  90. airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
  91. airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
  92. airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
  93. airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
  94. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  95. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  96. airbyte_cdk/sources/declarative/models/__init__.py +1 -1
  97. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1329 -595
  98. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
  99. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
  100. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
  101. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1699 -226
  102. airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
  103. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  104. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  105. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
  106. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  107. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
  108. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
  109. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
  110. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
  111. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
  112. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
  113. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
  114. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
  115. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
  116. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
  117. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
  118. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
  119. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  120. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
  121. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
  122. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
  123. airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
  124. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
  125. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
  126. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
  127. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
  128. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
  129. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
  130. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
  131. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
  132. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
  133. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
  134. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
  135. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
  136. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  137. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
  138. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
  139. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
  140. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
  141. airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
  142. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  143. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  144. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  145. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  146. airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
  147. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
  148. airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
  149. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +228 -72
  150. airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
  151. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
  152. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
  153. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
  154. airbyte_cdk/sources/declarative/spec/spec.py +12 -5
  155. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
  156. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
  157. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
  158. airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
  159. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  160. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  161. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  162. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  163. airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
  164. airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
  165. airbyte_cdk/sources/declarative/types.py +19 -110
  166. airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
  167. airbyte_cdk/sources/embedded/base_integration.py +16 -5
  168. airbyte_cdk/sources/embedded/catalog.py +16 -4
  169. airbyte_cdk/sources/embedded/runner.py +19 -3
  170. airbyte_cdk/sources/embedded/tools.py +5 -2
  171. airbyte_cdk/sources/file_based/README.md +152 -0
  172. airbyte_cdk/sources/file_based/__init__.py +24 -0
  173. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
  174. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
  175. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
  176. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +58 -10
  177. airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
  178. airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
  179. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  180. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
  181. airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
  182. airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
  183. airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
  184. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
  185. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
  186. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
  187. airbyte_cdk/sources/file_based/exceptions.py +52 -15
  188. airbyte_cdk/sources/file_based/file_based_source.py +163 -33
  189. airbyte_cdk/sources/file_based/file_based_stream_reader.py +83 -5
  190. airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
  191. airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
  192. airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
  193. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  194. airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
  195. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
  196. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
  197. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
  198. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +145 -41
  199. airbyte_cdk/sources/file_based/remote_file.py +1 -1
  200. airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
  201. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
  202. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
  203. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
  204. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
  205. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
  206. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
  207. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
  208. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
  209. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
  210. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
  211. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
  212. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +175 -45
  213. airbyte_cdk/sources/http_logger.py +8 -3
  214. airbyte_cdk/sources/message/__init__.py +7 -1
  215. airbyte_cdk/sources/message/repository.py +18 -4
  216. airbyte_cdk/sources/source.py +42 -38
  217. airbyte_cdk/sources/streams/__init__.py +2 -2
  218. airbyte_cdk/sources/streams/availability_strategy.py +54 -3
  219. airbyte_cdk/sources/streams/call_rate.py +64 -21
  220. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  221. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  222. airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
  223. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  224. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  225. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  226. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  227. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
  228. airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
  229. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
  230. airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
  231. airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
  232. airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
  233. airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
  234. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
  235. airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
  236. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
  237. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  238. airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
  239. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
  240. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
  241. airbyte_cdk/sources/streams/core.py +412 -87
  242. airbyte_cdk/sources/streams/http/__init__.py +2 -1
  243. airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
  244. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  245. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  246. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  247. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  248. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  249. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  250. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  251. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  252. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  253. airbyte_cdk/sources/streams/http/exceptions.py +27 -7
  254. airbyte_cdk/sources/streams/http/http.py +369 -246
  255. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  256. airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
  257. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
  258. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
  259. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
  260. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
  261. airbyte_cdk/sources/types.py +154 -0
  262. airbyte_cdk/sources/utils/record_helper.py +36 -21
  263. airbyte_cdk/sources/utils/schema_helpers.py +13 -6
  264. airbyte_cdk/sources/utils/slice_logger.py +4 -1
  265. airbyte_cdk/sources/utils/transform.py +54 -20
  266. airbyte_cdk/sql/_util/hashing.py +34 -0
  267. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  268. airbyte_cdk/sql/constants.py +32 -0
  269. airbyte_cdk/sql/exceptions.py +235 -0
  270. airbyte_cdk/sql/secrets.py +123 -0
  271. airbyte_cdk/sql/shared/__init__.py +15 -0
  272. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  273. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  274. airbyte_cdk/sql/types.py +160 -0
  275. airbyte_cdk/test/catalog_builder.py +70 -18
  276. airbyte_cdk/test/entrypoint_wrapper.py +117 -42
  277. airbyte_cdk/test/mock_http/__init__.py +1 -1
  278. airbyte_cdk/test/mock_http/matcher.py +6 -0
  279. airbyte_cdk/test/mock_http/mocker.py +57 -10
  280. airbyte_cdk/test/mock_http/request.py +19 -3
  281. airbyte_cdk/test/mock_http/response.py +3 -1
  282. airbyte_cdk/test/mock_http/response_builder.py +32 -16
  283. airbyte_cdk/test/state_builder.py +18 -10
  284. airbyte_cdk/test/utils/__init__.py +1 -0
  285. airbyte_cdk/test/utils/data.py +24 -0
  286. airbyte_cdk/test/utils/http_mocking.py +16 -0
  287. airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
  288. airbyte_cdk/test/utils/reading.py +26 -0
  289. airbyte_cdk/utils/__init__.py +2 -1
  290. airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
  291. airbyte_cdk/utils/analytics_message.py +10 -2
  292. airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
  293. airbyte_cdk/utils/event_timing.py +10 -10
  294. airbyte_cdk/utils/mapping_helpers.py +3 -1
  295. airbyte_cdk/utils/message_utils.py +20 -11
  296. airbyte_cdk/utils/print_buffer.py +75 -0
  297. airbyte_cdk/utils/schema_inferrer.py +198 -28
  298. airbyte_cdk/utils/slice_hasher.py +30 -0
  299. airbyte_cdk/utils/spec_schema_transformations.py +6 -3
  300. airbyte_cdk/utils/stream_status_utils.py +8 -1
  301. airbyte_cdk/utils/traced_exception.py +61 -21
  302. airbyte_cdk-6.13.1.dev4106.dist-info/METADATA +109 -0
  303. airbyte_cdk-6.13.1.dev4106.dist-info/RECORD +349 -0
  304. {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.13.1.dev4106.dist-info}/WHEEL +1 -2
  305. airbyte_cdk-6.13.1.dev4106.dist-info/entry_points.txt +3 -0
  306. airbyte_cdk/sources/declarative/create_partial.py +0 -92
  307. airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
  308. airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
  309. airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
  310. airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
  311. airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
  312. airbyte_cdk/sources/deprecated/base_source.py +0 -94
  313. airbyte_cdk/sources/deprecated/client.py +0 -99
  314. airbyte_cdk/sources/singer/__init__.py +0 -8
  315. airbyte_cdk/sources/singer/singer_helpers.py +0 -304
  316. airbyte_cdk/sources/singer/source.py +0 -186
  317. airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
  318. airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
  319. airbyte_cdk/sources/streams/http/auth/core.py +0 -29
  320. airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
  321. airbyte_cdk/sources/streams/http/auth/token.py +0 -47
  322. airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
  323. airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
  324. airbyte_cdk/sources/utils/schema_models.py +0 -84
  325. airbyte_cdk-0.72.0.dist-info/METADATA +0 -243
  326. airbyte_cdk-0.72.0.dist-info/RECORD +0 -466
  327. airbyte_cdk-0.72.0.dist-info/top_level.txt +0 -3
  328. source_declarative_manifest/main.py +0 -29
  329. unit_tests/connector_builder/__init__.py +0 -3
  330. unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
  331. unit_tests/connector_builder/test_message_grouper.py +0 -713
  332. unit_tests/connector_builder/utils.py +0 -27
  333. unit_tests/destinations/test_destination.py +0 -243
  334. unit_tests/singer/test_singer_helpers.py +0 -56
  335. unit_tests/singer/test_singer_source.py +0 -112
  336. unit_tests/sources/__init__.py +0 -0
  337. unit_tests/sources/concurrent_source/__init__.py +0 -3
  338. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
  339. unit_tests/sources/declarative/__init__.py +0 -3
  340. unit_tests/sources/declarative/auth/__init__.py +0 -3
  341. unit_tests/sources/declarative/auth/test_oauth.py +0 -331
  342. unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
  343. unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
  344. unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
  345. unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
  346. unit_tests/sources/declarative/checks/__init__.py +0 -3
  347. unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
  348. unit_tests/sources/declarative/decoders/__init__.py +0 -0
  349. unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
  350. unit_tests/sources/declarative/external_component.py +0 -13
  351. unit_tests/sources/declarative/extractors/__init__.py +0 -3
  352. unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
  353. unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
  354. unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
  355. unit_tests/sources/declarative/incremental/__init__.py +0 -0
  356. unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
  357. unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
  358. unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
  359. unit_tests/sources/declarative/interpolation/__init__.py +0 -3
  360. unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
  361. unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
  362. unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
  363. unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
  364. unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
  365. unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
  366. unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
  367. unit_tests/sources/declarative/parsers/__init__.py +0 -3
  368. unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
  369. unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
  370. unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1841
  371. unit_tests/sources/declarative/parsers/testing_components.py +0 -36
  372. unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
  373. unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
  374. unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
  375. unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
  376. unit_tests/sources/declarative/requesters/__init__.py +0 -3
  377. unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
  378. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
  379. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
  380. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
  381. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
  382. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
  383. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
  384. unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
  385. unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
  386. unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
  387. unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
  388. unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
  389. unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
  390. unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
  391. unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
  392. unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
  393. unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
  394. unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
  395. unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
  396. unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
  397. unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
  398. unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
  399. unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
  400. unit_tests/sources/declarative/retrievers/__init__.py +0 -3
  401. unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
  402. unit_tests/sources/declarative/schema/__init__.py +0 -6
  403. unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
  404. unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
  405. unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
  406. unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
  407. unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
  408. unit_tests/sources/declarative/states/__init__.py +0 -3
  409. unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
  410. unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
  411. unit_tests/sources/declarative/test_create_partial.py +0 -83
  412. unit_tests/sources/declarative/test_declarative_stream.py +0 -103
  413. unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
  414. unit_tests/sources/declarative/test_types.py +0 -39
  415. unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
  416. unit_tests/sources/file_based/__init__.py +0 -0
  417. unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
  418. unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
  419. unit_tests/sources/file_based/config/__init__.py +0 -0
  420. unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
  421. unit_tests/sources/file_based/config/test_csv_format.py +0 -34
  422. unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
  423. unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
  424. unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
  425. unit_tests/sources/file_based/file_types/__init__.py +0 -0
  426. unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
  427. unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
  428. unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
  429. unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
  430. unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
  431. unit_tests/sources/file_based/helpers.py +0 -70
  432. unit_tests/sources/file_based/in_memory_files_source.py +0 -211
  433. unit_tests/sources/file_based/scenarios/__init__.py +0 -0
  434. unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
  435. unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
  436. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
  437. unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
  438. unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
  439. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
  440. unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
  441. unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
  442. unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
  443. unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
  444. unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
  445. unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
  446. unit_tests/sources/file_based/stream/__init__.py +0 -0
  447. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  448. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
  449. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
  450. unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
  451. unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
  452. unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
  453. unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
  454. unit_tests/sources/file_based/test_scenarios.py +0 -253
  455. unit_tests/sources/file_based/test_schema_helpers.py +0 -346
  456. unit_tests/sources/fixtures/__init__.py +0 -3
  457. unit_tests/sources/fixtures/source_test_fixture.py +0 -153
  458. unit_tests/sources/message/__init__.py +0 -0
  459. unit_tests/sources/message/test_repository.py +0 -153
  460. unit_tests/sources/streams/__init__.py +0 -0
  461. unit_tests/sources/streams/concurrent/__init__.py +0 -3
  462. unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
  463. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
  464. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
  465. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
  466. unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
  467. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
  468. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
  469. unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
  470. unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
  471. unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
  472. unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
  473. unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
  474. unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
  475. unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
  476. unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
  477. unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
  478. unit_tests/sources/streams/http/__init__.py +0 -0
  479. unit_tests/sources/streams/http/auth/__init__.py +0 -0
  480. unit_tests/sources/streams/http/auth/test_auth.py +0 -173
  481. unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
  482. unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
  483. unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
  484. unit_tests/sources/streams/http/test_http.py +0 -635
  485. unit_tests/sources/streams/test_availability_strategy.py +0 -70
  486. unit_tests/sources/streams/test_call_rate.py +0 -300
  487. unit_tests/sources/streams/test_stream_read.py +0 -405
  488. unit_tests/sources/streams/test_streams_core.py +0 -184
  489. unit_tests/sources/test_abstract_source.py +0 -1442
  490. unit_tests/sources/test_concurrent_source.py +0 -112
  491. unit_tests/sources/test_config.py +0 -92
  492. unit_tests/sources/test_connector_state_manager.py +0 -482
  493. unit_tests/sources/test_http_logger.py +0 -252
  494. unit_tests/sources/test_integration_source.py +0 -86
  495. unit_tests/sources/test_source.py +0 -684
  496. unit_tests/sources/test_source_read.py +0 -460
  497. unit_tests/test/__init__.py +0 -0
  498. unit_tests/test/mock_http/__init__.py +0 -0
  499. unit_tests/test/mock_http/test_matcher.py +0 -53
  500. unit_tests/test/mock_http/test_mocker.py +0 -214
  501. unit_tests/test/mock_http/test_request.py +0 -117
  502. unit_tests/test/mock_http/test_response_builder.py +0 -177
  503. unit_tests/test/test_entrypoint_wrapper.py +0 -240
  504. unit_tests/utils/__init__.py +0 -0
  505. unit_tests/utils/test_datetime_format_inferrer.py +0 -60
  506. unit_tests/utils/test_mapping_helpers.py +0 -54
  507. unit_tests/utils/test_message_utils.py +0 -91
  508. unit_tests/utils/test_rate_limiting.py +0 -26
  509. unit_tests/utils/test_schema_inferrer.py +0 -202
  510. unit_tests/utils/test_secret_utils.py +0 -135
  511. unit_tests/utils/test_stream_status_utils.py +0 -61
  512. unit_tests/utils/test_traced_exception.py +0 -107
  513. /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
  514. {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
  515. {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
  516. {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
  517. {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.13.1.dev4106.dist-info}/LICENSE.txt +0 -0
@@ -1,214 +0,0 @@
1
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
-
3
- from unittest import TestCase
4
-
5
- import pytest
6
- import requests
7
- from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
8
-
9
- # Ensure that the scheme is HTTP as requests only partially supports other schemes
10
- # see https://github.com/psf/requests/blob/0b4d494192de489701d3a2e32acef8fb5d3f042e/src/requests/models.py#L424-L429
11
- _A_URL = "http://test.com/"
12
- _ANOTHER_URL = "http://another-test.com/"
13
- _A_RESPONSE_BODY = "a body"
14
- _ANOTHER_RESPONSE_BODY = "another body"
15
- _A_RESPONSE = HttpResponse("any response")
16
- _SOME_QUERY_PARAMS = {"q1": "query value"}
17
- _SOME_HEADERS = {"h1": "header value"}
18
- _OTHER_HEADERS = {"h2": "another header value"}
19
- _SOME_REQUEST_BODY_MAPPING = {"first_field": "first_value", "second_field": 2}
20
- _SOME_REQUEST_BODY_STR = "some_request_body"
21
-
22
-
23
- class HttpMockerTest(TestCase):
24
- @HttpMocker()
25
- def test_given_get_request_match_when_decorate_then_return_response(self, http_mocker):
26
- http_mocker.get(
27
- HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
28
- HttpResponse(_A_RESPONSE_BODY, 474, _OTHER_HEADERS),
29
- )
30
-
31
- response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
32
-
33
- assert response.text == _A_RESPONSE_BODY
34
- assert response.status_code == 474
35
- assert response.headers == _OTHER_HEADERS
36
-
37
- @HttpMocker()
38
- def test_given_loose_headers_matching_when_decorate_then_match(self, http_mocker):
39
- http_mocker.get(
40
- HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
41
- HttpResponse(_A_RESPONSE_BODY, 474),
42
- )
43
-
44
- requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS | {"more strict query param key": "any value"})
45
-
46
- @HttpMocker()
47
- def test_given_post_request_match_when_decorate_then_return_response(self, http_mocker):
48
- http_mocker.post(
49
- HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS, _SOME_REQUEST_BODY_STR),
50
- HttpResponse(_A_RESPONSE_BODY, 474),
51
- )
52
-
53
- response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR)
54
-
55
- assert response.text == _A_RESPONSE_BODY
56
- assert response.status_code == 474
57
-
58
- @HttpMocker()
59
- def test_given_multiple_responses_when_decorate_get_request_then_return_response(self, http_mocker):
60
- http_mocker.get(
61
- HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
62
- [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)],
63
- )
64
-
65
- first_response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
66
- second_response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
67
-
68
- assert first_response.text == _A_RESPONSE_BODY
69
- assert first_response.status_code == 1
70
- assert second_response.text == _ANOTHER_RESPONSE_BODY
71
- assert second_response.status_code == 2
72
-
73
- @HttpMocker()
74
- def test_given_multiple_responses_when_decorate_post_request_then_return_response(self, http_mocker):
75
- http_mocker.post(
76
- HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS, _SOME_REQUEST_BODY_STR),
77
- [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)],
78
- )
79
-
80
- first_response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR)
81
- second_response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR)
82
-
83
- assert first_response.text == _A_RESPONSE_BODY
84
- assert first_response.status_code == 1
85
- assert second_response.text == _ANOTHER_RESPONSE_BODY
86
- assert second_response.status_code == 2
87
-
88
- @HttpMocker()
89
- def test_given_more_requests_than_responses_when_decorate_then_raise_error(self, http_mocker):
90
- http_mocker.get(
91
- HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
92
- [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)],
93
- )
94
-
95
- last_response = [requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) for _ in range(10)][-1]
96
-
97
- assert last_response.text == _ANOTHER_RESPONSE_BODY
98
- assert last_response.status_code == 2
99
-
100
- @HttpMocker()
101
- def test_given_all_requests_match_when_decorate_then_do_not_raise(self, http_mocker):
102
- http_mocker.get(
103
- HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS),
104
- _A_RESPONSE,
105
- )
106
- requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
107
-
108
- def test_given_missing_requests_when_decorate_then_raise(self):
109
- @HttpMocker()
110
- def decorated_function(http_mocker):
111
- http_mocker.get(
112
- HttpRequest(_A_URL),
113
- _A_RESPONSE,
114
- )
115
-
116
- with pytest.raises(ValueError) as exc_info:
117
- decorated_function()
118
- assert "Invalid number of matches" in str(exc_info.value)
119
-
120
- def test_given_assertion_error_when_decorate_then_raise_assertion_error(self):
121
- @HttpMocker()
122
- def decorated_function(http_mocker):
123
- http_mocker.get(
124
- HttpRequest(_A_URL),
125
- _A_RESPONSE,
126
- )
127
- requests.get(_A_URL)
128
- assert False
129
-
130
- with pytest.raises(AssertionError):
131
- decorated_function()
132
-
133
- def test_given_assertion_error_but_missing_request_when_decorate_then_raise_missing_http_request(self):
134
- @HttpMocker()
135
- def decorated_function(http_mocker):
136
- http_mocker.get(
137
- HttpRequest(_A_URL),
138
- _A_RESPONSE,
139
- )
140
- assert False
141
-
142
- with pytest.raises(ValueError) as exc_info:
143
- decorated_function()
144
- assert "Invalid number of matches" in str(exc_info.value)
145
-
146
- def test_given_request_does_not_match_when_decorate_then_raise(self):
147
- @HttpMocker()
148
- def decorated_function(http_mocker):
149
- http_mocker.get(
150
- HttpRequest(_A_URL),
151
- _A_RESPONSE,
152
- )
153
- requests.get(_ANOTHER_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS)
154
-
155
- with pytest.raises(ValueError) as exc_info:
156
- decorated_function()
157
- assert "No matcher matches" in str(exc_info.value)
158
-
159
- def test_given_request_matches_multiple_matchers_when_decorate_then_match_first_one(self):
160
- less_granular_headers = {"less_granular": "1"}
161
- more_granular_headers = {"more_granular": "2"} | less_granular_headers
162
-
163
- @HttpMocker()
164
- def decorated_function(http_mocker):
165
- http_mocker.get(
166
- HttpRequest(_A_URL, headers=more_granular_headers),
167
- _A_RESPONSE,
168
- )
169
- http_mocker.get(
170
- HttpRequest(_A_URL, headers=less_granular_headers),
171
- _A_RESPONSE,
172
- )
173
- requests.get(_A_URL, headers=more_granular_headers)
174
-
175
- with pytest.raises(ValueError) as exc_info:
176
- decorated_function()
177
- assert "more_granular" in str(exc_info.value) # the matcher corresponding to the first `http_mocker.get` is not matched
178
-
179
- def test_given_exact_number_of_call_provided_when_assert_number_of_calls_then_do_not_raise(self):
180
- @HttpMocker()
181
- def decorated_function(http_mocker):
182
- request = HttpRequest(_A_URL)
183
- http_mocker.get(request, _A_RESPONSE)
184
-
185
- requests.get(_A_URL)
186
- requests.get(_A_URL)
187
-
188
- http_mocker.assert_number_of_calls(request, 2)
189
-
190
- decorated_function()
191
- # then do not raise
192
-
193
- def test_given_invalid_number_of_call_provided_when_assert_number_of_calls_then_raise(self):
194
- @HttpMocker()
195
- def decorated_function(http_mocker):
196
- request = HttpRequest(_A_URL)
197
- http_mocker.get(request, _A_RESPONSE)
198
-
199
- requests.get(_A_URL)
200
- requests.get(_A_URL)
201
-
202
- http_mocker.assert_number_of_calls(request, 1)
203
-
204
- with pytest.raises(AssertionError):
205
- decorated_function()
206
-
207
- def test_given_unknown_request_when_assert_number_of_calls_then_raise(self):
208
- @HttpMocker()
209
- def decorated_function(http_mocker):
210
- http_mocker.get(HttpRequest(_A_URL), _A_RESPONSE)
211
- http_mocker.assert_number_of_calls(HttpRequest(_ANOTHER_URL), 1)
212
-
213
- with pytest.raises(ValueError):
214
- decorated_function()
@@ -1,117 +0,0 @@
1
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
-
3
- from unittest import TestCase
4
-
5
- import pytest
6
- from airbyte_cdk.test.mock_http.request import ANY_QUERY_PARAMS, HttpRequest
7
-
8
-
9
- class HttpRequestMatcherTest(TestCase):
10
- def test_given_query_params_as_dict_and_string_then_query_params_are_properly_considered(self):
11
- with_string = HttpRequest("mock://test.com/path", query_params="a_query_param=q1&a_list_param=first&a_list_param=second")
12
- with_dict = HttpRequest("mock://test.com/path", query_params={"a_query_param": "q1", "a_list_param": ["first", "second"]})
13
- assert with_string.matches(with_dict) and with_dict.matches(with_string)
14
-
15
- def test_given_query_params_in_url_and_also_provided_then_raise_error(self):
16
- with pytest.raises(ValueError):
17
- HttpRequest("mock://test.com/path?a_query_param=1", query_params={"another_query_param": "2"})
18
-
19
- def test_given_same_url_query_params_and_subset_headers_when_matches_then_return_true(self):
20
- request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}, {"first_header": "h1"})
21
- actual_request = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}, {"first_header": "h1", "second_header": "h2"})
22
- assert actual_request.matches(request_to_match)
23
-
24
- def test_given_url_differs_when_matches_then_return_false(self):
25
- assert not HttpRequest("mock://test.com/another_path").matches(HttpRequest("mock://test.com/path"))
26
-
27
- def test_given_query_params_differs_when_matches_then_return_false(self):
28
- request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"})
29
- actual_request = HttpRequest("mock://test.com/path", {"another_query_param": "q2"})
30
- assert not actual_request.matches(request_to_match)
31
-
32
- def test_given_query_params_is_subset_differs_when_matches_then_return_false(self):
33
- request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"})
34
- actual_request = HttpRequest("mock://test.com/path", {"a_query_param": "q1", "another_query_param": "q2"})
35
- assert not actual_request.matches(request_to_match)
36
-
37
- def test_given_headers_is_subset_differs_when_matches_then_return_true(self):
38
- request_to_match = HttpRequest("mock://test.com/path", headers={"first_header": "h1"})
39
- actual_request = HttpRequest("mock://test.com/path", headers={"first_header": "h1", "second_header": "h2"})
40
- assert actual_request.matches(request_to_match)
41
-
42
- def test_given_headers_value_does_not_match_differs_when_matches_then_return_false(self):
43
- request_to_match = HttpRequest("mock://test.com/path", headers={"first_header": "h1"})
44
- actual_request = HttpRequest("mock://test.com/path", headers={"first_header": "value does not match"})
45
- assert not actual_request.matches(request_to_match)
46
-
47
- def test_given_same_body_mappings_value_when_matches_then_return_true(self):
48
- request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value", "second_field": 2})
49
- actual_request = HttpRequest("mock://test.com/path", body={"first_field": "first_value", "second_field": 2})
50
- assert actual_request.matches(request_to_match)
51
-
52
- def test_given_bodies_are_mapping_and_differs_when_matches_then_return_false(self):
53
- request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"})
54
- actual_request = HttpRequest("mock://test.com/path", body={"first_field": "value does not match"})
55
- assert not actual_request.matches(request_to_match)
56
-
57
- def test_given_same_mapping_and_bytes_when_matches_then_return_true(self):
58
- request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"})
59
- actual_request = HttpRequest("mock://test.com/path", body=b'{"first_field": "first_value"}')
60
- assert actual_request.matches(request_to_match)
61
-
62
- def test_given_different_mapping_and_bytes_when_matches_then_return_false(self):
63
- request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"})
64
- actual_request = HttpRequest("mock://test.com/path", body=b'{"first_field": "another value"}')
65
- assert not actual_request.matches(request_to_match)
66
-
67
- def test_given_same_mapping_and_str_when_matches_then_return_true(self):
68
- request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"})
69
- actual_request = HttpRequest("mock://test.com/path", body='{"first_field": "first_value"}')
70
- assert actual_request.matches(request_to_match)
71
-
72
- def test_given_different_mapping_and_str_when_matches_then_return_false(self):
73
- request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"})
74
- actual_request = HttpRequest("mock://test.com/path", body='{"first_field": "another value"}')
75
- assert not actual_request.matches(request_to_match)
76
-
77
- def test_given_same_bytes_and_mapping_when_matches_then_return_true(self):
78
- request_to_match = HttpRequest("mock://test.com/path", body=b'{"first_field": "first_value"}')
79
- actual_request = HttpRequest("mock://test.com/path", body={"first_field": "first_value"})
80
- assert actual_request.matches(request_to_match)
81
-
82
- def test_given_different_bytes_and_mapping_when_matches_then_return_false(self):
83
- request_to_match = HttpRequest("mock://test.com/path", body=b'{"first_field": "first_value"}')
84
- actual_request = HttpRequest("mock://test.com/path", body={"first_field": "another value"})
85
- assert not actual_request.matches(request_to_match)
86
-
87
- def test_given_same_str_and_mapping_when_matches_then_return_true(self):
88
- request_to_match = HttpRequest("mock://test.com/path", body='{"first_field": "first_value"}')
89
- actual_request = HttpRequest("mock://test.com/path", body={"first_field": "first_value"})
90
- assert actual_request.matches(request_to_match)
91
-
92
- def test_given_different_str_and_mapping_when_matches_then_return_false(self):
93
- request_to_match = HttpRequest("mock://test.com/path", body='{"first_field": "first_value"}')
94
- actual_request = HttpRequest("mock://test.com/path", body={"first_field": "another value"})
95
- assert not actual_request.matches(request_to_match)
96
-
97
- def test_given_same_body_str_value_when_matches_then_return_true(self):
98
- request_to_match = HttpRequest("mock://test.com/path", body="some_request_body")
99
- actual_request = HttpRequest("mock://test.com/path", body="some_request_body")
100
- assert actual_request.matches(request_to_match)
101
-
102
- def test_given_body_str_value_differs_when_matches_then_return_false(self):
103
- request_to_match = HttpRequest("mock://test.com/path", body="some_request_body")
104
- actual_request = HttpRequest("mock://test.com/path", body="another_request_body")
105
- assert not actual_request.matches(request_to_match)
106
-
107
- def test_given_any_matcher_for_query_param_when_matches_then_return_true(self):
108
- request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"})
109
- actual_request = HttpRequest("mock://test.com/path", ANY_QUERY_PARAMS)
110
-
111
- assert actual_request.matches(request_to_match)
112
- assert request_to_match.matches(actual_request)
113
-
114
- def test_given_any_matcher_for_both_when_matches_then_return_true(self):
115
- request_to_match = HttpRequest("mock://test.com/path", ANY_QUERY_PARAMS)
116
- actual_request = HttpRequest("mock://test.com/path", ANY_QUERY_PARAMS)
117
- assert actual_request.matches(request_to_match)
@@ -1,177 +0,0 @@
1
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
- import json
3
- from copy import deepcopy
4
- from pathlib import Path as FilePath
5
- from typing import Any, Dict, Optional, Union
6
- from unittest import TestCase
7
- from unittest.mock import Mock
8
-
9
- import pytest
10
- from airbyte_cdk.test.mock_http.response import HttpResponse
11
- from airbyte_cdk.test.mock_http.response_builder import (
12
- FieldPath,
13
- FieldUpdatePaginationStrategy,
14
- HttpResponseBuilder,
15
- NestedPath,
16
- PaginationStrategy,
17
- Path,
18
- RecordBuilder,
19
- create_record_builder,
20
- create_response_builder,
21
- find_template,
22
- )
23
-
24
- _RECORDS_FIELD = "records_field"
25
- _ID_FIELD = "record_id"
26
- _CURSOR_FIELD = "record_cursor"
27
- _ANY_RECORD = {"a_record_field": "a record value"}
28
- _SOME_RECORDS = {_RECORDS_FIELD: [_ANY_RECORD]}
29
- _A_RESPONSE_TEMPLATE = _SOME_RECORDS
30
-
31
- _RECORD_BUILDER = 0
32
- _RESPONSE_BUILDER = 1
33
-
34
-
35
- def _record_builder(
36
- response_template: Dict[str, Any],
37
- records_path: Union[FieldPath, NestedPath],
38
- record_id_path: Optional[Path] = None,
39
- record_cursor_path: Optional[Union[FieldPath, NestedPath]] = None,
40
- ) -> RecordBuilder:
41
- return create_record_builder(deepcopy(response_template), records_path, record_id_path, record_cursor_path)
42
-
43
-
44
- def _any_record_builder() -> RecordBuilder:
45
- return create_record_builder({"record_path": [{"a_record": "record value"}]}, FieldPath("record_path"))
46
-
47
-
48
- def _response_builder(
49
- response_template: Dict[str, Any],
50
- records_path: Union[FieldPath, NestedPath],
51
- pagination_strategy: Optional[PaginationStrategy] = None
52
- ) -> HttpResponseBuilder:
53
- return create_response_builder(deepcopy(response_template), records_path, pagination_strategy=pagination_strategy)
54
-
55
-
56
- def _body(response: HttpResponse) -> Dict[str, Any]:
57
- return json.loads(response.body)
58
-
59
-
60
- class RecordBuilderTest(TestCase):
61
- def test_given_with_id_when_build_then_set_id(self) -> None:
62
- builder = _record_builder({_RECORDS_FIELD: [{_ID_FIELD: "an id"}]}, FieldPath(_RECORDS_FIELD), FieldPath(_ID_FIELD))
63
- record = builder.with_id("another id").build()
64
- assert record[_ID_FIELD] == "another id"
65
-
66
- def test_given_nested_id_when_build_then_set_id(self) -> None:
67
- builder = _record_builder({_RECORDS_FIELD: [{"nested": {_ID_FIELD: "id"}}]}, FieldPath(_RECORDS_FIELD), NestedPath(["nested", _ID_FIELD]))
68
- record = builder.with_id("another id").build()
69
- assert record["nested"][_ID_FIELD] == "another id"
70
-
71
- def test_given_id_path_not_provided_but_with_id_when_build_then_raise_error(self) -> None:
72
- builder = _record_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD), None)
73
- with pytest.raises(ValueError):
74
- builder.with_id("any_id").build()
75
-
76
- def test_given_no_id_in_template_for_path_when_build_then_raise_error(self) -> None:
77
- with pytest.raises(ValueError):
78
- _record_builder({_RECORDS_FIELD: [{"record without id": "should fail"}]}, FieldPath(_RECORDS_FIELD), FieldPath(_ID_FIELD))
79
-
80
- def test_given_with_cursor_when_build_then_set_id(self) -> None:
81
- builder = _record_builder(
82
- {_RECORDS_FIELD: [{_CURSOR_FIELD: "a cursor"}]},
83
- FieldPath(_RECORDS_FIELD),
84
- record_cursor_path=FieldPath(_CURSOR_FIELD)
85
- )
86
- record = builder.with_cursor("another cursor").build()
87
- assert record[_CURSOR_FIELD] == "another cursor"
88
-
89
- def test_given_nested_cursor_when_build_then_set_cursor(self) -> None:
90
- builder = _record_builder(
91
- {_RECORDS_FIELD: [{"nested": {_CURSOR_FIELD: "a cursor"}}]},
92
- FieldPath(_RECORDS_FIELD),
93
- record_cursor_path=NestedPath(["nested", _CURSOR_FIELD])
94
- )
95
- record = builder.with_cursor("another cursor").build()
96
- assert record["nested"][_CURSOR_FIELD] == "another cursor"
97
-
98
- def test_given_with_field_when_build_then_write_field(self) -> None:
99
- builder = _any_record_builder()
100
- record = builder.with_field(FieldPath("to_write_field"), "a field value").build()
101
- assert record["to_write_field"] == "a field value"
102
-
103
- def test_given_nested_cursor_when_build_then_write_field(self) -> None:
104
- builder = _any_record_builder()
105
- record = builder.with_field(NestedPath(["path", "to_write_field"]), "a field value").build()
106
- assert record["path"]["to_write_field"] == "a field value"
107
-
108
- def test_given_cursor_path_not_provided_but_with_id_when_build_then_raise_error(self) -> None:
109
- builder = _record_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD))
110
- with pytest.raises(ValueError):
111
- builder.with_cursor("any cursor").build()
112
-
113
- def test_given_no_cursor_in_template_for_path_when_build_then_raise_error(self) -> None:
114
- with pytest.raises(ValueError):
115
- _record_builder(
116
- {_RECORDS_FIELD: [{"record without cursor": "should fail"}]},
117
- FieldPath(_RECORDS_FIELD),
118
- record_cursor_path=FieldPath(_ID_FIELD)
119
- )
120
-
121
-
122
- class HttpResponseBuilderTest(TestCase):
123
- def test_given_records_in_template_but_no_with_records_when_build_then_no_records(self) -> None:
124
- builder = _response_builder({_RECORDS_FIELD: [{"a_record_field": "a record value"}]}, FieldPath(_RECORDS_FIELD))
125
- response = builder.build()
126
- assert len(_body(response)[_RECORDS_FIELD]) == 0
127
-
128
- def test_given_many_records_when_build_then_response_has_records(self) -> None:
129
- builder = _response_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD))
130
- a_record_builder = Mock(spec=RecordBuilder)
131
- a_record_builder.build.return_value = {"a record": 1}
132
- another_record_builder = Mock(spec=RecordBuilder)
133
- another_record_builder.build.return_value = {"another record": 2}
134
-
135
- response = builder.with_record(a_record_builder).with_record(another_record_builder).build()
136
-
137
- assert len(_body(response)[_RECORDS_FIELD]) == 2
138
-
139
- def test_when_build_then_default_status_code_is_200(self) -> None:
140
- builder = _response_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD))
141
- response = builder.build()
142
- assert response.status_code == 200
143
-
144
- def test_given_status_code_when_build_then_status_code_is_set(self) -> None:
145
- builder = _response_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD))
146
- response = builder.with_status_code(239).build()
147
- assert response.status_code == 239
148
-
149
- def test_given_pagination_with_strategy_when_build_then_apply_strategy(self) -> None:
150
- builder = _response_builder(
151
- {"has_more_pages": False} | _SOME_RECORDS,
152
- FieldPath(_RECORDS_FIELD),
153
- pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more_pages"), "yes more page")
154
- )
155
-
156
- response = builder.with_pagination().build()
157
-
158
- assert _body(response)["has_more_pages"] == "yes more page"
159
-
160
- def test_given_no_pagination_strategy_but_pagination_when_build_then_raise_error(self) -> None:
161
- builder = _response_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD))
162
- with pytest.raises(ValueError):
163
- builder.with_pagination()
164
-
165
-
166
- class UtilMethodsTest(TestCase):
167
- def test_from_resource_file(self) -> None:
168
- template = find_template("test-resource", __file__)
169
- assert template == {"test-source template": "this is a template for test-resource"}
170
-
171
- def test_given_cwd_doesnt_have_unit_tests_as_parent_when_from_resource_file__then_raise_error(self) -> None:
172
- with pytest.raises(ValueError):
173
- find_template("test-resource", str(FilePath(__file__).parent.parent.parent.parent))
174
-
175
- def test_given_records_path_invalid_when_create_builders_from_resource_then_raise_exception(self) -> None:
176
- with pytest.raises(ValueError):
177
- create_record_builder(_A_RESPONSE_TEMPLATE, NestedPath(["invalid", "record", "path"]))