airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev1__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (518) hide show
  1. airbyte_cdk/__init__.py +355 -6
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +29 -10
  7. airbyte_cdk/connector.py +24 -24
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
  10. airbyte_cdk/connector_builder/main.py +45 -13
  11. airbyte_cdk/connector_builder/message_grouper.py +189 -50
  12. airbyte_cdk/connector_builder/models.py +3 -2
  13. airbyte_cdk/destinations/__init__.py +4 -3
  14. airbyte_cdk/destinations/destination.py +54 -20
  15. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  16. airbyte_cdk/destinations/vector_db_based/config.py +40 -17
  17. airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
  18. airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
  19. airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
  20. airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
  21. airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
  22. airbyte_cdk/entrypoint.py +153 -44
  23. airbyte_cdk/exception_handler.py +21 -3
  24. airbyte_cdk/logger.py +30 -44
  25. airbyte_cdk/models/__init__.py +13 -2
  26. airbyte_cdk/models/airbyte_protocol.py +86 -1
  27. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  28. airbyte_cdk/models/file_transfer_record_message.py +13 -0
  29. airbyte_cdk/models/well_known_types.py +1 -1
  30. airbyte_cdk/sources/__init__.py +5 -1
  31. airbyte_cdk/sources/abstract_source.py +125 -79
  32. airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
  33. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
  34. airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
  35. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
  36. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  37. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
  38. airbyte_cdk/sources/config.py +3 -2
  39. airbyte_cdk/sources/connector_state_manager.py +49 -83
  40. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  41. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
  42. airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
  43. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  44. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  45. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  46. airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
  47. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
  48. airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
  49. airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
  50. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
  51. airbyte_cdk/sources/declarative/auth/token.py +28 -10
  52. airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
  53. airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
  54. airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
  55. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  56. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  57. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
  58. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
  59. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
  60. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1185 -85
  61. airbyte_cdk/sources/declarative/declarative_source.py +5 -2
  62. airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
  63. airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
  64. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
  65. airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
  66. airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
  67. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  68. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  69. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  70. airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
  71. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
  72. airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
  73. airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
  74. airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
  75. airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
  76. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
  77. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  78. airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
  79. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +340 -0
  80. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
  81. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  82. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
  83. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +174 -74
  84. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  85. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  86. airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
  87. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
  88. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
  89. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
  90. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
  91. airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
  92. airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
  93. airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
  94. airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
  95. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  96. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  97. airbyte_cdk/sources/declarative/models/__init__.py +1 -1
  98. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1319 -603
  99. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
  100. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
  101. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
  102. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1759 -225
  103. airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
  104. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  105. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  106. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
  107. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  108. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
  109. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
  110. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
  111. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
  112. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
  113. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
  114. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
  115. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
  116. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
  117. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
  118. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
  119. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
  120. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  121. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
  122. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
  123. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
  124. airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
  125. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
  126. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
  127. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
  128. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
  129. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
  130. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
  131. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
  132. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
  133. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
  134. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
  135. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
  136. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
  137. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  138. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
  139. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
  140. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
  141. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
  142. airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
  143. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  144. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  145. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  146. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  147. airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
  148. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
  149. airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
  150. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
  151. airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
  152. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
  153. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
  154. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
  155. airbyte_cdk/sources/declarative/spec/spec.py +12 -5
  156. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
  157. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
  158. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
  159. airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
  160. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  161. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  162. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  163. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  164. airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
  165. airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
  166. airbyte_cdk/sources/declarative/types.py +19 -110
  167. airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
  168. airbyte_cdk/sources/embedded/base_integration.py +16 -5
  169. airbyte_cdk/sources/embedded/catalog.py +16 -4
  170. airbyte_cdk/sources/embedded/runner.py +19 -3
  171. airbyte_cdk/sources/embedded/tools.py +5 -2
  172. airbyte_cdk/sources/file_based/README.md +152 -0
  173. airbyte_cdk/sources/file_based/__init__.py +24 -0
  174. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
  175. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
  176. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
  177. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
  178. airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
  179. airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
  180. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  181. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
  182. airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
  183. airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
  184. airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
  185. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
  186. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
  187. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
  188. airbyte_cdk/sources/file_based/exceptions.py +18 -15
  189. airbyte_cdk/sources/file_based/file_based_source.py +140 -33
  190. airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
  191. airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
  192. airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
  193. airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
  194. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  195. airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
  196. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
  197. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
  198. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
  199. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
  200. airbyte_cdk/sources/file_based/remote_file.py +1 -1
  201. airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
  202. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
  203. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
  204. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
  205. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
  206. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
  207. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
  208. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
  209. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
  210. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
  211. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
  212. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
  213. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
  214. airbyte_cdk/sources/http_logger.py +8 -3
  215. airbyte_cdk/sources/message/__init__.py +7 -1
  216. airbyte_cdk/sources/message/repository.py +18 -4
  217. airbyte_cdk/sources/source.py +42 -38
  218. airbyte_cdk/sources/streams/__init__.py +2 -2
  219. airbyte_cdk/sources/streams/availability_strategy.py +54 -3
  220. airbyte_cdk/sources/streams/call_rate.py +64 -21
  221. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  222. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  223. airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
  224. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  225. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  226. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  227. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  228. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
  229. airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
  230. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
  231. airbyte_cdk/sources/streams/concurrent/cursor.py +313 -48
  232. airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
  233. airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
  234. airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
  235. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
  236. airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
  237. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
  238. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  239. airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
  240. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
  241. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
  242. airbyte_cdk/sources/streams/core.py +412 -87
  243. airbyte_cdk/sources/streams/http/__init__.py +2 -1
  244. airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
  245. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  246. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  247. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  248. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  249. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  250. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  251. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  252. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  253. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  254. airbyte_cdk/sources/streams/http/exceptions.py +27 -7
  255. airbyte_cdk/sources/streams/http/http.py +369 -246
  256. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  257. airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
  258. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
  259. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
  260. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
  261. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
  262. airbyte_cdk/sources/types.py +154 -0
  263. airbyte_cdk/sources/utils/record_helper.py +36 -21
  264. airbyte_cdk/sources/utils/schema_helpers.py +13 -6
  265. airbyte_cdk/sources/utils/slice_logger.py +4 -1
  266. airbyte_cdk/sources/utils/transform.py +54 -20
  267. airbyte_cdk/sql/_util/hashing.py +34 -0
  268. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  269. airbyte_cdk/sql/constants.py +32 -0
  270. airbyte_cdk/sql/exceptions.py +235 -0
  271. airbyte_cdk/sql/secrets.py +123 -0
  272. airbyte_cdk/sql/shared/__init__.py +15 -0
  273. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  274. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  275. airbyte_cdk/sql/types.py +160 -0
  276. airbyte_cdk/test/catalog_builder.py +70 -18
  277. airbyte_cdk/test/entrypoint_wrapper.py +117 -42
  278. airbyte_cdk/test/mock_http/__init__.py +1 -1
  279. airbyte_cdk/test/mock_http/matcher.py +6 -0
  280. airbyte_cdk/test/mock_http/mocker.py +57 -10
  281. airbyte_cdk/test/mock_http/request.py +19 -3
  282. airbyte_cdk/test/mock_http/response.py +3 -1
  283. airbyte_cdk/test/mock_http/response_builder.py +32 -16
  284. airbyte_cdk/test/state_builder.py +18 -10
  285. airbyte_cdk/test/utils/__init__.py +1 -0
  286. airbyte_cdk/test/utils/data.py +24 -0
  287. airbyte_cdk/test/utils/http_mocking.py +16 -0
  288. airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
  289. airbyte_cdk/test/utils/reading.py +26 -0
  290. airbyte_cdk/utils/__init__.py +2 -1
  291. airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
  292. airbyte_cdk/utils/analytics_message.py +10 -2
  293. airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
  294. airbyte_cdk/utils/event_timing.py +10 -10
  295. airbyte_cdk/utils/mapping_helpers.py +3 -1
  296. airbyte_cdk/utils/message_utils.py +20 -11
  297. airbyte_cdk/utils/print_buffer.py +75 -0
  298. airbyte_cdk/utils/schema_inferrer.py +198 -28
  299. airbyte_cdk/utils/slice_hasher.py +30 -0
  300. airbyte_cdk/utils/spec_schema_transformations.py +6 -3
  301. airbyte_cdk/utils/stream_status_utils.py +8 -1
  302. airbyte_cdk/utils/traced_exception.py +61 -21
  303. airbyte_cdk-6.17.1.dev1.dist-info/METADATA +109 -0
  304. airbyte_cdk-6.17.1.dev1.dist-info/RECORD +350 -0
  305. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev1.dist-info}/WHEEL +1 -2
  306. airbyte_cdk-6.17.1.dev1.dist-info/entry_points.txt +3 -0
  307. airbyte_cdk/sources/declarative/create_partial.py +0 -92
  308. airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
  309. airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
  310. airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
  311. airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
  312. airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
  313. airbyte_cdk/sources/deprecated/base_source.py +0 -94
  314. airbyte_cdk/sources/deprecated/client.py +0 -99
  315. airbyte_cdk/sources/singer/__init__.py +0 -8
  316. airbyte_cdk/sources/singer/singer_helpers.py +0 -304
  317. airbyte_cdk/sources/singer/source.py +0 -186
  318. airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
  319. airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
  320. airbyte_cdk/sources/streams/http/auth/core.py +0 -29
  321. airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
  322. airbyte_cdk/sources/streams/http/auth/token.py +0 -47
  323. airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
  324. airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
  325. airbyte_cdk/sources/utils/schema_models.py +0 -84
  326. airbyte_cdk-0.72.1.dist-info/METADATA +0 -243
  327. airbyte_cdk-0.72.1.dist-info/RECORD +0 -466
  328. airbyte_cdk-0.72.1.dist-info/top_level.txt +0 -3
  329. source_declarative_manifest/main.py +0 -29
  330. unit_tests/connector_builder/__init__.py +0 -3
  331. unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
  332. unit_tests/connector_builder/test_message_grouper.py +0 -713
  333. unit_tests/connector_builder/utils.py +0 -27
  334. unit_tests/destinations/test_destination.py +0 -243
  335. unit_tests/singer/test_singer_helpers.py +0 -56
  336. unit_tests/singer/test_singer_source.py +0 -112
  337. unit_tests/sources/__init__.py +0 -0
  338. unit_tests/sources/concurrent_source/__init__.py +0 -3
  339. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
  340. unit_tests/sources/declarative/__init__.py +0 -3
  341. unit_tests/sources/declarative/auth/__init__.py +0 -3
  342. unit_tests/sources/declarative/auth/test_oauth.py +0 -331
  343. unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
  344. unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
  345. unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
  346. unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
  347. unit_tests/sources/declarative/checks/__init__.py +0 -3
  348. unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
  349. unit_tests/sources/declarative/decoders/__init__.py +0 -0
  350. unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
  351. unit_tests/sources/declarative/external_component.py +0 -13
  352. unit_tests/sources/declarative/extractors/__init__.py +0 -3
  353. unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
  354. unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
  355. unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
  356. unit_tests/sources/declarative/incremental/__init__.py +0 -0
  357. unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
  358. unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
  359. unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
  360. unit_tests/sources/declarative/interpolation/__init__.py +0 -3
  361. unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
  362. unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
  363. unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
  364. unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
  365. unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
  366. unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
  367. unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
  368. unit_tests/sources/declarative/parsers/__init__.py +0 -3
  369. unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
  370. unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
  371. unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1847
  372. unit_tests/sources/declarative/parsers/testing_components.py +0 -36
  373. unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
  374. unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
  375. unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
  376. unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
  377. unit_tests/sources/declarative/requesters/__init__.py +0 -3
  378. unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
  379. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
  380. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
  381. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
  382. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
  383. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
  384. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
  385. unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
  386. unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
  387. unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
  388. unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
  389. unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
  390. unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
  391. unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
  392. unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
  393. unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
  394. unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
  395. unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
  396. unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
  397. unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
  398. unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
  399. unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
  400. unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
  401. unit_tests/sources/declarative/retrievers/__init__.py +0 -3
  402. unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
  403. unit_tests/sources/declarative/schema/__init__.py +0 -6
  404. unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
  405. unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
  406. unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
  407. unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
  408. unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
  409. unit_tests/sources/declarative/states/__init__.py +0 -3
  410. unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
  411. unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
  412. unit_tests/sources/declarative/test_create_partial.py +0 -83
  413. unit_tests/sources/declarative/test_declarative_stream.py +0 -103
  414. unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
  415. unit_tests/sources/declarative/test_types.py +0 -39
  416. unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
  417. unit_tests/sources/file_based/__init__.py +0 -0
  418. unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
  419. unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
  420. unit_tests/sources/file_based/config/__init__.py +0 -0
  421. unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
  422. unit_tests/sources/file_based/config/test_csv_format.py +0 -34
  423. unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
  424. unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
  425. unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
  426. unit_tests/sources/file_based/file_types/__init__.py +0 -0
  427. unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
  428. unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
  429. unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
  430. unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
  431. unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
  432. unit_tests/sources/file_based/helpers.py +0 -70
  433. unit_tests/sources/file_based/in_memory_files_source.py +0 -211
  434. unit_tests/sources/file_based/scenarios/__init__.py +0 -0
  435. unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
  436. unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
  437. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
  438. unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
  439. unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
  440. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
  441. unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
  442. unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
  443. unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
  444. unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
  445. unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
  446. unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
  447. unit_tests/sources/file_based/stream/__init__.py +0 -0
  448. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  449. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
  450. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
  451. unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
  452. unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
  453. unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
  454. unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
  455. unit_tests/sources/file_based/test_scenarios.py +0 -253
  456. unit_tests/sources/file_based/test_schema_helpers.py +0 -346
  457. unit_tests/sources/fixtures/__init__.py +0 -3
  458. unit_tests/sources/fixtures/source_test_fixture.py +0 -153
  459. unit_tests/sources/message/__init__.py +0 -0
  460. unit_tests/sources/message/test_repository.py +0 -153
  461. unit_tests/sources/streams/__init__.py +0 -0
  462. unit_tests/sources/streams/concurrent/__init__.py +0 -3
  463. unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
  464. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
  465. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
  466. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
  467. unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
  468. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
  469. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
  470. unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
  471. unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
  472. unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
  473. unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
  474. unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
  475. unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
  476. unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
  477. unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
  478. unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
  479. unit_tests/sources/streams/http/__init__.py +0 -0
  480. unit_tests/sources/streams/http/auth/__init__.py +0 -0
  481. unit_tests/sources/streams/http/auth/test_auth.py +0 -173
  482. unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
  483. unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
  484. unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
  485. unit_tests/sources/streams/http/test_http.py +0 -635
  486. unit_tests/sources/streams/test_availability_strategy.py +0 -70
  487. unit_tests/sources/streams/test_call_rate.py +0 -300
  488. unit_tests/sources/streams/test_stream_read.py +0 -405
  489. unit_tests/sources/streams/test_streams_core.py +0 -184
  490. unit_tests/sources/test_abstract_source.py +0 -1442
  491. unit_tests/sources/test_concurrent_source.py +0 -112
  492. unit_tests/sources/test_config.py +0 -92
  493. unit_tests/sources/test_connector_state_manager.py +0 -482
  494. unit_tests/sources/test_http_logger.py +0 -252
  495. unit_tests/sources/test_integration_source.py +0 -86
  496. unit_tests/sources/test_source.py +0 -684
  497. unit_tests/sources/test_source_read.py +0 -460
  498. unit_tests/test/__init__.py +0 -0
  499. unit_tests/test/mock_http/__init__.py +0 -0
  500. unit_tests/test/mock_http/test_matcher.py +0 -53
  501. unit_tests/test/mock_http/test_mocker.py +0 -214
  502. unit_tests/test/mock_http/test_request.py +0 -117
  503. unit_tests/test/mock_http/test_response_builder.py +0 -177
  504. unit_tests/test/test_entrypoint_wrapper.py +0 -240
  505. unit_tests/utils/__init__.py +0 -0
  506. unit_tests/utils/test_datetime_format_inferrer.py +0 -60
  507. unit_tests/utils/test_mapping_helpers.py +0 -54
  508. unit_tests/utils/test_message_utils.py +0 -91
  509. unit_tests/utils/test_rate_limiting.py +0 -26
  510. unit_tests/utils/test_schema_inferrer.py +0 -202
  511. unit_tests/utils/test_secret_utils.py +0 -135
  512. unit_tests/utils/test_stream_status_utils.py +0 -61
  513. unit_tests/utils/test_traced_exception.py +0 -107
  514. /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
  515. {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
  516. {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
  517. {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
  518. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev1.dist-info}/LICENSE.txt +0 -0
@@ -1,423 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- import json
6
- import logging
7
- from typing import Optional, Union
8
- from unittest.mock import Mock
9
-
10
- import freezegun
11
- import pendulum
12
- import pytest
13
- import requests
14
- from airbyte_cdk.models import FailureType, OrchestratorType, Type
15
- from airbyte_cdk.sources.streams.http.requests_native_auth import (
16
- BasicHttpAuthenticator,
17
- MultipleTokenAuthenticator,
18
- Oauth2Authenticator,
19
- SingleUseRefreshTokenOauth2Authenticator,
20
- TokenAuthenticator,
21
- )
22
- from airbyte_cdk.utils import AirbyteTracedException
23
- from requests import Response
24
- from requests.exceptions import RequestException
25
-
26
- LOGGER = logging.getLogger(__name__)
27
-
28
- resp = Response()
29
-
30
-
31
- def test_token_authenticator():
32
- """
33
- Should match passed in token, no matter how many times token is retrieved.
34
- """
35
- token_auth = TokenAuthenticator(token="test-token")
36
- header1 = token_auth.get_auth_header()
37
- header2 = token_auth.get_auth_header()
38
-
39
- prepared_request = requests.PreparedRequest()
40
- prepared_request.headers = {}
41
- token_auth(prepared_request)
42
-
43
- assert {"Authorization": "Bearer test-token"} == prepared_request.headers
44
- assert {"Authorization": "Bearer test-token"} == header1
45
- assert {"Authorization": "Bearer test-token"} == header2
46
-
47
-
48
- def test_basic_http_authenticator():
49
- """
50
- Should match passed in token, no matter how many times token is retrieved.
51
- """
52
- token_auth = BasicHttpAuthenticator(username="user", password="password")
53
- header1 = token_auth.get_auth_header()
54
- header2 = token_auth.get_auth_header()
55
-
56
- prepared_request = requests.PreparedRequest()
57
- prepared_request.headers = {}
58
- token_auth(prepared_request)
59
-
60
- assert {"Authorization": "Basic dXNlcjpwYXNzd29yZA=="} == prepared_request.headers
61
- assert {"Authorization": "Basic dXNlcjpwYXNzd29yZA=="} == header1
62
- assert {"Authorization": "Basic dXNlcjpwYXNzd29yZA=="} == header2
63
-
64
-
65
- def test_multiple_token_authenticator():
66
- multiple_token_auth = MultipleTokenAuthenticator(tokens=["token1", "token2"])
67
- header1 = multiple_token_auth.get_auth_header()
68
- header2 = multiple_token_auth.get_auth_header()
69
- header3 = multiple_token_auth.get_auth_header()
70
-
71
- prepared_request = requests.PreparedRequest()
72
- prepared_request.headers = {}
73
- multiple_token_auth(prepared_request)
74
-
75
- assert {"Authorization": "Bearer token2"} == prepared_request.headers
76
- assert {"Authorization": "Bearer token1"} == header1
77
- assert {"Authorization": "Bearer token2"} == header2
78
- assert {"Authorization": "Bearer token1"} == header3
79
-
80
-
81
- class TestOauth2Authenticator:
82
- """
83
- Test class for OAuth2Authenticator.
84
- """
85
-
86
- refresh_endpoint = "refresh_end"
87
- client_id = "client_id"
88
- client_secret = "client_secret"
89
- refresh_token = "refresh_token"
90
-
91
- def test_get_auth_header_fresh(self, mocker):
92
- """
93
- Should not retrieve new token if current token is valid.
94
- """
95
- oauth = Oauth2Authenticator(
96
- token_refresh_endpoint=TestOauth2Authenticator.refresh_endpoint,
97
- client_id=TestOauth2Authenticator.client_id,
98
- client_secret=TestOauth2Authenticator.client_secret,
99
- refresh_token=TestOauth2Authenticator.refresh_token,
100
- )
101
-
102
- mocker.patch.object(Oauth2Authenticator, "refresh_access_token", return_value=("access_token", 1000))
103
- header = oauth.get_auth_header()
104
- assert {"Authorization": "Bearer access_token"} == header
105
-
106
- def test_get_auth_header_expired(self, mocker):
107
- """
108
- Should retrieve new token if current token is expired.
109
- """
110
- oauth = Oauth2Authenticator(
111
- token_refresh_endpoint=TestOauth2Authenticator.refresh_endpoint,
112
- client_id=TestOauth2Authenticator.client_id,
113
- client_secret=TestOauth2Authenticator.client_secret,
114
- refresh_token=TestOauth2Authenticator.refresh_token,
115
- )
116
-
117
- expire_immediately = 0
118
- mocker.patch.object(Oauth2Authenticator, "refresh_access_token", return_value=("access_token_1", expire_immediately))
119
- oauth.get_auth_header() # Set the first expired token.
120
-
121
- valid_100_secs = 100
122
- mocker.patch.object(Oauth2Authenticator, "refresh_access_token", return_value=("access_token_2", valid_100_secs))
123
- header = oauth.get_auth_header()
124
- assert {"Authorization": "Bearer access_token_2"} == header
125
-
126
- def test_refresh_request_body(self):
127
- """
128
- Request body should match given configuration.
129
- """
130
- scopes = ["scope1", "scope2"]
131
- oauth = Oauth2Authenticator(
132
- token_refresh_endpoint="refresh_end",
133
- client_id="some_client_id",
134
- client_secret="some_client_secret",
135
- refresh_token="some_refresh_token",
136
- scopes=["scope1", "scope2"],
137
- token_expiry_date=pendulum.now().add(days=3),
138
- grant_type="some_grant_type",
139
- refresh_request_body={"custom_field": "in_outbound_request", "another_field": "exists_in_body", "scopes": ["no_override"]},
140
- )
141
- body = oauth.build_refresh_request_body()
142
- expected = {
143
- "grant_type": "some_grant_type",
144
- "client_id": "some_client_id",
145
- "client_secret": "some_client_secret",
146
- "refresh_token": "some_refresh_token",
147
- "scopes": scopes,
148
- "custom_field": "in_outbound_request",
149
- "another_field": "exists_in_body",
150
- }
151
- assert body == expected
152
-
153
- def test_refresh_access_token(self, mocker):
154
- oauth = Oauth2Authenticator(
155
- token_refresh_endpoint="refresh_end",
156
- client_id="some_client_id",
157
- client_secret="some_client_secret",
158
- refresh_token="some_refresh_token",
159
- scopes=["scope1", "scope2"],
160
- token_expiry_date=pendulum.now().add(days=3),
161
- refresh_request_body={"custom_field": "in_outbound_request", "another_field": "exists_in_body", "scopes": ["no_override"]},
162
- )
163
-
164
- resp.status_code = 200
165
- mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": 1000})
166
- mocker.patch.object(requests, "request", side_effect=mock_request, autospec=True)
167
- token, expires_in = oauth.refresh_access_token()
168
-
169
- assert isinstance(expires_in, int)
170
- assert ("access_token", 1000) == (token, expires_in)
171
-
172
- # Test with expires_in as str
173
- mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": "2000"})
174
- token, expires_in = oauth.refresh_access_token()
175
-
176
- assert isinstance(expires_in, str)
177
- assert ("access_token", "2000") == (token, expires_in)
178
-
179
- # Test with expires_in as str
180
- mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": "2022-04-24T00:00:00Z"})
181
- token, expires_in = oauth.refresh_access_token()
182
-
183
- assert isinstance(expires_in, str)
184
- assert ("access_token", "2022-04-24T00:00:00Z") == (token, expires_in)
185
-
186
- @pytest.mark.parametrize(
187
- "expires_in_response, token_expiry_date_format, expected_token_expiry_date",
188
- [
189
- (3600, None, pendulum.datetime(year=2022, month=1, day=1, hour=1)),
190
- ("90012", None, pendulum.datetime(year=2022, month=1, day=2, hour=1, second=12)),
191
- ("2024-02-28", "YYYY-MM-DD", pendulum.datetime(year=2024, month=2, day=28)),
192
- ("2022-02-12T00:00:00.000000+00:00", "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", pendulum.datetime(year=2022, month=2, day=12)),
193
- ],
194
- ids=["seconds", "string_of_seconds", "simple_date", "simple_datetime"],
195
- )
196
- @freezegun.freeze_time("2022-01-01")
197
- def test_parse_refresh_token_lifespan(
198
- self,
199
- mocker,
200
- expires_in_response: Union[str, int],
201
- token_expiry_date_format: Optional[str],
202
- expected_token_expiry_date: pendulum.DateTime,
203
- ):
204
- oauth = Oauth2Authenticator(
205
- token_refresh_endpoint="refresh_end",
206
- client_id="some_client_id",
207
- client_secret="some_client_secret",
208
- refresh_token="some_refresh_token",
209
- scopes=["scope1", "scope2"],
210
- token_expiry_date=pendulum.now().subtract(days=3),
211
- token_expiry_date_format=token_expiry_date_format,
212
- token_expiry_is_time_of_expiration=bool(token_expiry_date_format),
213
- refresh_request_body={"custom_field": "in_outbound_request", "another_field": "exists_in_body", "scopes": ["no_override"]},
214
- )
215
-
216
- resp.status_code = 200
217
- mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": expires_in_response})
218
- mocker.patch.object(requests, "request", side_effect=mock_request, autospec=True)
219
- token, expire_in = oauth.refresh_access_token()
220
- expires_datetime = oauth._parse_token_expiration_date(expire_in)
221
-
222
- assert isinstance(expires_datetime, pendulum.DateTime)
223
- assert ("access_token", expected_token_expiry_date) == (token, expires_datetime)
224
-
225
- @pytest.mark.parametrize("error_code", (429, 500, 502, 504))
226
- def test_refresh_access_token_retry(self, error_code, requests_mock):
227
- oauth = Oauth2Authenticator(
228
- f"https://{TestOauth2Authenticator.refresh_endpoint}",
229
- TestOauth2Authenticator.client_id,
230
- TestOauth2Authenticator.client_secret,
231
- TestOauth2Authenticator.refresh_token,
232
- )
233
- requests_mock.post(
234
- f"https://{TestOauth2Authenticator.refresh_endpoint}",
235
- [{"status_code": error_code}, {"status_code": error_code}, {"json": {"access_token": "token", "expires_in": 10}}],
236
- )
237
- token, expires_in = oauth.refresh_access_token()
238
- assert isinstance(expires_in, int)
239
- assert (token, expires_in) == ("token", 10)
240
- assert requests_mock.call_count == 3
241
-
242
- def test_auth_call_method(self, mocker):
243
- oauth = Oauth2Authenticator(
244
- token_refresh_endpoint=TestOauth2Authenticator.refresh_endpoint,
245
- client_id=TestOauth2Authenticator.client_id,
246
- client_secret=TestOauth2Authenticator.client_secret,
247
- refresh_token=TestOauth2Authenticator.refresh_token,
248
- )
249
-
250
- mocker.patch.object(Oauth2Authenticator, "refresh_access_token", return_value=("access_token", 1000))
251
- prepared_request = requests.PreparedRequest()
252
- prepared_request.headers = {}
253
- oauth(prepared_request)
254
-
255
- assert {"Authorization": "Bearer access_token"} == prepared_request.headers
256
-
257
- @pytest.mark.parametrize(
258
- ("config_codes", "response_code", "config_key", "response_key", "config_values", "response_value", "wrapped"),
259
- (
260
- ((400,), 400, "error", "error", ("invalid_grant",), "invalid_grant", True),
261
- ((401,), 400, "error", "error", ("invalid_grant",), "invalid_grant", False),
262
- ((400,), 400, "error_key", "error", ("invalid_grant",), "invalid_grant", False),
263
- ((400,), 400, "error", "error", ("invalid_grant",), "valid_grant", False),
264
- ((), 400, "", "error", (), "valid_grant", False),
265
- ),
266
- )
267
- def test_refresh_access_token_wrapped(
268
- self, requests_mock, config_codes, response_code, config_key, response_key, config_values, response_value, wrapped
269
- ):
270
- oauth = Oauth2Authenticator(
271
- f"https://{TestOauth2Authenticator.refresh_endpoint}",
272
- TestOauth2Authenticator.client_id,
273
- TestOauth2Authenticator.client_secret,
274
- TestOauth2Authenticator.refresh_token,
275
- refresh_token_error_status_codes=config_codes,
276
- refresh_token_error_key=config_key,
277
- refresh_token_error_values=config_values,
278
- )
279
- error_content = {response_key: response_value}
280
- requests_mock.post(f"https://{TestOauth2Authenticator.refresh_endpoint}", status_code=response_code, json=error_content)
281
-
282
- exception_to_raise = AirbyteTracedException if wrapped else RequestException
283
- with pytest.raises(exception_to_raise) as exc_info:
284
- oauth.refresh_access_token()
285
-
286
- if wrapped:
287
- error_message = "Refresh token is invalid or expired. Please re-authenticate from Sources/<your source>/Settings."
288
- assert exc_info.value.internal_message == error_message
289
- assert exc_info.value.message == error_message
290
- assert exc_info.value.failure_type == FailureType.config_error
291
-
292
-
293
- class TestSingleUseRefreshTokenOauth2Authenticator:
294
- @pytest.fixture
295
- def connector_config(self):
296
- return {
297
- "credentials": {
298
- "access_token": "my_access_token",
299
- "refresh_token": "my_refresh_token",
300
- "client_id": "my_client_id",
301
- "client_secret": "my_client_secret",
302
- "token_expiry_date": "2022-12-31T00:00:00+00:00",
303
- }
304
- }
305
-
306
- @pytest.fixture
307
- def invalid_connector_config(self):
308
- return {"no_credentials_key": "foo"}
309
-
310
- def test_init(self, connector_config):
311
- authenticator = SingleUseRefreshTokenOauth2Authenticator(
312
- connector_config,
313
- token_refresh_endpoint="foobar",
314
- client_id=connector_config["credentials"]["client_id"],
315
- client_secret=connector_config["credentials"]["client_secret"],
316
- )
317
- assert authenticator.access_token == connector_config["credentials"]["access_token"]
318
- assert authenticator.get_refresh_token() == connector_config["credentials"]["refresh_token"]
319
- assert authenticator.get_token_expiry_date() == pendulum.parse(connector_config["credentials"]["token_expiry_date"])
320
-
321
- @freezegun.freeze_time("2022-12-31")
322
- @pytest.mark.parametrize(
323
- "test_name, expires_in_value, expiry_date_format, expected_expiry_date",
324
- [
325
- ("number_of_seconds", 42, None, "2022-12-31T00:00:42+00:00"),
326
- ("string_of_seconds", "42", None, "2022-12-31T00:00:42+00:00"),
327
- ("date_format", "2023-04-04", "YYYY-MM-DD", "2023-04-04T00:00:00+00:00"),
328
- ],
329
- )
330
- def test_given_no_message_repository_get_access_token(
331
- self, test_name, expires_in_value, expiry_date_format, expected_expiry_date, capsys, mocker, connector_config
332
- ):
333
- authenticator = SingleUseRefreshTokenOauth2Authenticator(
334
- connector_config,
335
- token_refresh_endpoint="foobar",
336
- client_id=connector_config["credentials"]["client_id"],
337
- client_secret=connector_config["credentials"]["client_secret"],
338
- token_expiry_date_format=expiry_date_format,
339
- )
340
- authenticator.refresh_access_token = mocker.Mock(return_value=("new_access_token", expires_in_value, "new_refresh_token"))
341
- authenticator.token_has_expired = mocker.Mock(return_value=True)
342
- access_token = authenticator.get_access_token()
343
- captured = capsys.readouterr()
344
- airbyte_message = json.loads(captured.out)
345
- expected_new_config = connector_config.copy()
346
- expected_new_config["credentials"]["access_token"] = "new_access_token"
347
- expected_new_config["credentials"]["refresh_token"] = "new_refresh_token"
348
- expected_new_config["credentials"]["token_expiry_date"] = expected_expiry_date
349
- assert airbyte_message["control"]["connectorConfig"]["config"] == expected_new_config
350
- assert authenticator.access_token == access_token == "new_access_token"
351
- assert authenticator.get_refresh_token() == "new_refresh_token"
352
- assert authenticator.get_token_expiry_date() > pendulum.now()
353
- authenticator.token_has_expired = mocker.Mock(return_value=False)
354
- access_token = authenticator.get_access_token()
355
- captured = capsys.readouterr()
356
- assert not captured.out
357
- assert authenticator.access_token == access_token == "new_access_token"
358
-
359
- def test_given_message_repository_when_get_access_token_then_emit_message(self, mocker, connector_config):
360
- message_repository = Mock()
361
- authenticator = SingleUseRefreshTokenOauth2Authenticator(
362
- connector_config,
363
- token_refresh_endpoint="foobar",
364
- client_id=connector_config["credentials"]["client_id"],
365
- client_secret=connector_config["credentials"]["client_secret"],
366
- token_expiry_date_format="YYYY-MM-DD",
367
- message_repository=message_repository,
368
- )
369
- authenticator.refresh_access_token = mocker.Mock(return_value=("new_access_token", "2023-04-04", "new_refresh_token"))
370
- authenticator.token_has_expired = mocker.Mock(return_value=True)
371
-
372
- authenticator.get_access_token()
373
-
374
- emitted_message = message_repository.emit_message.call_args_list[0].args[0]
375
- assert emitted_message.type == Type.CONTROL
376
- assert emitted_message.control.type == OrchestratorType.CONNECTOR_CONFIG
377
- assert emitted_message.control.connectorConfig.config["credentials"]["access_token"] == "new_access_token"
378
- assert emitted_message.control.connectorConfig.config["credentials"]["refresh_token"] == "new_refresh_token"
379
- assert emitted_message.control.connectorConfig.config["credentials"]["token_expiry_date"] == "2023-04-04T00:00:00+00:00"
380
- assert emitted_message.control.connectorConfig.config["credentials"]["client_id"] == "my_client_id"
381
- assert emitted_message.control.connectorConfig.config["credentials"]["client_secret"] == "my_client_secret"
382
-
383
- def test_given_message_repository_when_get_access_token_then_log_request(self, mocker, connector_config):
384
- message_repository = Mock()
385
- authenticator = SingleUseRefreshTokenOauth2Authenticator(
386
- connector_config,
387
- token_refresh_endpoint="foobar",
388
- client_id=connector_config["credentials"]["client_id"],
389
- client_secret=connector_config["credentials"]["client_secret"],
390
- message_repository=message_repository,
391
- )
392
- mocker.patch("airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth.requests.request")
393
- mocker.patch(
394
- "airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth.format_http_message", return_value="formatted json"
395
- )
396
- authenticator.token_has_expired = mocker.Mock(return_value=True)
397
-
398
- authenticator.get_access_token()
399
-
400
- assert message_repository.log_message.call_count == 1
401
-
402
- def test_refresh_access_token(self, mocker, connector_config):
403
- authenticator = SingleUseRefreshTokenOauth2Authenticator(
404
- connector_config,
405
- token_refresh_endpoint="foobar",
406
- client_id=connector_config["credentials"]["client_id"],
407
- client_secret=connector_config["credentials"]["client_secret"],
408
- )
409
-
410
- authenticator._get_refresh_access_token_response = mocker.Mock(
411
- return_value={
412
- authenticator.get_access_token_name(): "new_access_token",
413
- authenticator.get_expires_in_name(): "42",
414
- authenticator.get_refresh_token_name(): "new_refresh_token",
415
- }
416
- )
417
- assert authenticator.refresh_access_token() == ("new_access_token", "42", "new_refresh_token")
418
-
419
-
420
- def mock_request(method, url, data):
421
- if url == "refresh_end":
422
- return resp
423
- raise Exception(f"Error while refreshing access token with request: {method}, {url}, {data}")
@@ -1,180 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- import logging
6
- from typing import Any, Iterable, List, Mapping, Optional, Tuple
7
-
8
- import pytest
9
- import requests
10
- from airbyte_cdk.sources import AbstractSource
11
- from airbyte_cdk.sources.streams import Stream
12
- from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
13
- from airbyte_cdk.sources.streams.http.http import HttpStream
14
- from requests import HTTPError
15
-
16
- logger = logging.getLogger("airbyte")
17
-
18
-
19
- class MockHttpStream(HttpStream):
20
- url_base = "https://test_base_url.com"
21
- primary_key = ""
22
-
23
- def __init__(self, **kwargs):
24
- super().__init__(**kwargs)
25
- self.resp_counter = 1
26
-
27
- def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
28
- return None
29
-
30
- def path(self, **kwargs) -> str:
31
- return ""
32
-
33
- def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
34
- stub_resp = {"data": self.resp_counter}
35
- self.resp_counter += 1
36
- yield stub_resp
37
-
38
- pass
39
-
40
- def retry_factor(self) -> float:
41
- return 0.01
42
-
43
-
44
- @pytest.mark.parametrize(
45
- ("status_code", "json_contents", "expected_is_available", "expected_messages"),
46
- [
47
- (
48
- 403,
49
- {"error": "Something went wrong"},
50
- False,
51
- [
52
- "This is most likely due to insufficient permissions on the credentials in use.",
53
- "Something went wrong",
54
- ],
55
- ),
56
- (200, {}, True, []),
57
- ],
58
- )
59
- @pytest.mark.parametrize(
60
- ("include_source", "expected_docs_url_messages"),
61
- [
62
- (True, ["Please visit https://docs.airbyte.com/integrations/sources/MockSource to learn more."]),
63
- (False, ["Please visit the connector's documentation to learn more."]),
64
- ],
65
- )
66
- @pytest.mark.parametrize("records_as_list", [True, False])
67
- def test_default_http_availability_strategy(
68
- mocker,
69
- status_code,
70
- json_contents,
71
- expected_is_available,
72
- expected_messages,
73
- include_source,
74
- expected_docs_url_messages,
75
- records_as_list,
76
- ):
77
- class MockListHttpStream(MockHttpStream):
78
- def read_records(self, *args, **kvargs):
79
- if records_as_list:
80
- return list(super().read_records(*args, **kvargs))
81
- else:
82
- return super().read_records(*args, **kvargs)
83
-
84
- http_stream = MockListHttpStream()
85
- assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy)
86
-
87
- class MockResponseWithJsonContents(requests.Response, mocker.MagicMock):
88
- def __init__(self, *args, **kvargs):
89
- mocker.MagicMock.__init__(self)
90
- requests.Response.__init__(self, **kvargs)
91
- self.json = mocker.MagicMock()
92
-
93
- class MockSource(AbstractSource):
94
- def __init__(self, streams: List[Stream] = None):
95
- self._streams = streams
96
-
97
- def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
98
- return True, ""
99
-
100
- def streams(self, config: Mapping[str, Any]) -> List[Stream]:
101
- if not self._streams:
102
- raise Exception("Stream is not set")
103
- return self._streams
104
-
105
- response = MockResponseWithJsonContents()
106
- response.status_code = status_code
107
- response.json.return_value = json_contents
108
- mocker.patch.object(requests.Session, "send", return_value=response)
109
-
110
- if include_source:
111
- source = MockSource(streams=[http_stream])
112
- actual_is_available, reason = http_stream.check_availability(logger, source)
113
- else:
114
- actual_is_available, reason = http_stream.check_availability(logger)
115
-
116
- assert expected_is_available == actual_is_available
117
- if expected_is_available:
118
- assert reason is None
119
- else:
120
- all_expected_messages = expected_messages + expected_docs_url_messages
121
- for message in all_expected_messages:
122
- assert message in reason
123
-
124
-
125
- def test_http_availability_raises_unhandled_error(mocker):
126
- http_stream = MockHttpStream()
127
- assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy)
128
-
129
- req = requests.Response()
130
- req.status_code = 404
131
- mocker.patch.object(requests.Session, "send", return_value=req)
132
-
133
- with pytest.raises(HTTPError):
134
- http_stream.check_availability(logger)
135
-
136
-
137
- def test_send_handles_retries_when_checking_availability(mocker, caplog):
138
- http_stream = MockHttpStream()
139
- assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy)
140
-
141
- req_1 = requests.Response()
142
- req_1.status_code = 429
143
- req_2 = requests.Response()
144
- req_2.status_code = 503
145
- req_3 = requests.Response()
146
- req_3.status_code = 200
147
- mock_send = mocker.patch.object(requests.Session, "send", side_effect=[req_1, req_2, req_3])
148
-
149
- with caplog.at_level(logging.INFO):
150
- stream_is_available, _ = http_stream.check_availability(logger)
151
-
152
- assert stream_is_available
153
- assert mock_send.call_count == 3
154
- for message in ["Caught retryable error", "Response Code: 429", "Response Code: 503"]:
155
- assert message in caplog.text
156
-
157
-
158
- @pytest.mark.parametrize("records_as_list", [True, False])
159
- def test_http_availability_strategy_on_empty_stream(mocker, records_as_list):
160
- class MockEmptyHttpStream(mocker.MagicMock, MockHttpStream):
161
- def __init__(self, *args, **kvargs):
162
- mocker.MagicMock.__init__(self)
163
- self.read_records = mocker.MagicMock()
164
-
165
- empty_stream = MockEmptyHttpStream()
166
- assert isinstance(empty_stream, HttpStream)
167
-
168
- assert isinstance(empty_stream.availability_strategy, HttpAvailabilityStrategy)
169
-
170
- # Generator should have no values to generate
171
- if records_as_list:
172
- empty_stream.read_records.return_value = []
173
- else:
174
- empty_stream.read_records.return_value = iter([])
175
-
176
- logger = logging.getLogger("airbyte.test-source")
177
- stream_is_available, _ = empty_stream.check_availability(logger)
178
-
179
- assert stream_is_available
180
- assert empty_stream.read_records.called