airbyte-cdk 0.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (368) hide show
  1. airbyte_cdk/__init__.py +358 -0
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +236 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +104 -0
  7. airbyte_cdk/connector.py +123 -0
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/__init__.py +3 -0
  10. airbyte_cdk/connector_builder/connector_builder_handler.py +121 -0
  11. airbyte_cdk/connector_builder/main.py +107 -0
  12. airbyte_cdk/connector_builder/models.py +73 -0
  13. airbyte_cdk/connector_builder/test_reader/__init__.py +7 -0
  14. airbyte_cdk/connector_builder/test_reader/helpers.py +689 -0
  15. airbyte_cdk/connector_builder/test_reader/message_grouper.py +173 -0
  16. airbyte_cdk/connector_builder/test_reader/reader.py +441 -0
  17. airbyte_cdk/connector_builder/test_reader/types.py +83 -0
  18. airbyte_cdk/destinations/__init__.py +8 -0
  19. airbyte_cdk/destinations/destination.py +154 -0
  20. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  21. airbyte_cdk/destinations/vector_db_based/__init__.py +38 -0
  22. airbyte_cdk/destinations/vector_db_based/config.py +298 -0
  23. airbyte_cdk/destinations/vector_db_based/document_processor.py +223 -0
  24. airbyte_cdk/destinations/vector_db_based/embedder.py +303 -0
  25. airbyte_cdk/destinations/vector_db_based/indexer.py +78 -0
  26. airbyte_cdk/destinations/vector_db_based/test_utils.py +63 -0
  27. airbyte_cdk/destinations/vector_db_based/utils.py +35 -0
  28. airbyte_cdk/destinations/vector_db_based/writer.py +104 -0
  29. airbyte_cdk/entrypoint.py +414 -0
  30. airbyte_cdk/exception_handler.py +56 -0
  31. airbyte_cdk/logger.py +109 -0
  32. airbyte_cdk/models/__init__.py +72 -0
  33. airbyte_cdk/models/airbyte_protocol.py +88 -0
  34. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  35. airbyte_cdk/models/well_known_types.py +5 -0
  36. airbyte_cdk/py.typed +0 -0
  37. airbyte_cdk/sources/__init__.py +26 -0
  38. airbyte_cdk/sources/abstract_source.py +326 -0
  39. airbyte_cdk/sources/concurrent_source/__init__.py +8 -0
  40. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +255 -0
  41. airbyte_cdk/sources/concurrent_source/concurrent_source.py +165 -0
  42. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +147 -0
  43. airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py +24 -0
  44. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  45. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +115 -0
  46. airbyte_cdk/sources/config.py +27 -0
  47. airbyte_cdk/sources/connector_state_manager.py +161 -0
  48. airbyte_cdk/sources/declarative/__init__.py +3 -0
  49. airbyte_cdk/sources/declarative/async_job/__init__.py +0 -0
  50. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  51. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +525 -0
  52. airbyte_cdk/sources/declarative/async_job/job_tracker.py +79 -0
  53. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  54. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  55. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  56. airbyte_cdk/sources/declarative/auth/__init__.py +8 -0
  57. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +42 -0
  58. airbyte_cdk/sources/declarative/auth/jwt.py +197 -0
  59. airbyte_cdk/sources/declarative/auth/oauth.py +293 -0
  60. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +45 -0
  61. airbyte_cdk/sources/declarative/auth/token.py +267 -0
  62. airbyte_cdk/sources/declarative/auth/token_provider.py +82 -0
  63. airbyte_cdk/sources/declarative/checks/__init__.py +24 -0
  64. airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +61 -0
  65. airbyte_cdk/sources/declarative/checks/check_stream.py +56 -0
  66. airbyte_cdk/sources/declarative/checks/connection_checker.py +35 -0
  67. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  68. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  69. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +526 -0
  70. airbyte_cdk/sources/declarative/datetime/__init__.py +3 -0
  71. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +65 -0
  72. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +118 -0
  73. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +3975 -0
  74. airbyte_cdk/sources/declarative/declarative_source.py +36 -0
  75. airbyte_cdk/sources/declarative/declarative_stream.py +241 -0
  76. airbyte_cdk/sources/declarative/decoders/__init__.py +33 -0
  77. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +218 -0
  78. airbyte_cdk/sources/declarative/decoders/decoder.py +32 -0
  79. airbyte_cdk/sources/declarative/decoders/decoder_parser.py +30 -0
  80. airbyte_cdk/sources/declarative/decoders/json_decoder.py +65 -0
  81. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  82. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  83. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  84. airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py +56 -0
  85. airbyte_cdk/sources/declarative/exceptions.py +9 -0
  86. airbyte_cdk/sources/declarative/extractors/__init__.py +21 -0
  87. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +86 -0
  88. airbyte_cdk/sources/declarative/extractors/http_selector.py +37 -0
  89. airbyte_cdk/sources/declarative/extractors/record_extractor.py +27 -0
  90. airbyte_cdk/sources/declarative/extractors/record_filter.py +91 -0
  91. airbyte_cdk/sources/declarative/extractors/record_selector.py +170 -0
  92. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +176 -0
  93. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  94. airbyte_cdk/sources/declarative/incremental/__init__.py +37 -0
  95. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +497 -0
  96. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +459 -0
  97. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  98. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +357 -0
  99. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +380 -0
  100. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  101. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  102. airbyte_cdk/sources/declarative/interpolation/__init__.py +9 -0
  103. airbyte_cdk/sources/declarative/interpolation/filters.py +139 -0
  104. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +66 -0
  105. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +56 -0
  106. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +52 -0
  107. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +79 -0
  108. airbyte_cdk/sources/declarative/interpolation/interpolation.py +34 -0
  109. airbyte_cdk/sources/declarative/interpolation/jinja.py +161 -0
  110. airbyte_cdk/sources/declarative/interpolation/macros.py +191 -0
  111. airbyte_cdk/sources/declarative/manifest_declarative_source.py +421 -0
  112. airbyte_cdk/sources/declarative/migrations/__init__.py +0 -0
  113. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  114. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  115. airbyte_cdk/sources/declarative/models/__init__.py +2 -0
  116. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2503 -0
  117. airbyte_cdk/sources/declarative/parsers/__init__.py +3 -0
  118. airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +157 -0
  119. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +21 -0
  120. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +172 -0
  121. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +213 -0
  122. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +3407 -0
  123. airbyte_cdk/sources/declarative/partition_routers/__init__.py +29 -0
  124. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  125. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  126. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +121 -0
  127. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  128. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +63 -0
  129. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +437 -0
  130. airbyte_cdk/sources/declarative/requesters/README.md +56 -0
  131. airbyte_cdk/sources/declarative/requesters/__init__.py +9 -0
  132. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +25 -0
  133. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +23 -0
  134. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +45 -0
  135. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +45 -0
  136. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +41 -0
  137. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +70 -0
  138. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +77 -0
  139. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +17 -0
  140. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +101 -0
  141. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +147 -0
  142. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  143. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +17 -0
  144. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +179 -0
  145. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +350 -0
  146. airbyte_cdk/sources/declarative/requesters/http_requester.py +433 -0
  147. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +21 -0
  148. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +327 -0
  149. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +76 -0
  150. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +65 -0
  151. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +25 -0
  152. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +98 -0
  153. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +102 -0
  154. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +71 -0
  155. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +48 -0
  156. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +66 -0
  157. airbyte_cdk/sources/declarative/requesters/request_option.py +117 -0
  158. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +23 -0
  159. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +92 -0
  160. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  161. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +59 -0
  162. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +68 -0
  163. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +119 -0
  164. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +79 -0
  165. airbyte_cdk/sources/declarative/requesters/request_path.py +15 -0
  166. airbyte_cdk/sources/declarative/requesters/requester.py +144 -0
  167. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  168. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  169. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  170. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  171. airbyte_cdk/sources/declarative/retrievers/__init__.py +19 -0
  172. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +124 -0
  173. airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
  174. airbyte_cdk/sources/declarative/retrievers/retriever.py +54 -0
  175. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +702 -0
  176. airbyte_cdk/sources/declarative/schema/__init__.py +25 -0
  177. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +47 -0
  178. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +285 -0
  179. airbyte_cdk/sources/declarative/schema/inline_schema_loader.py +19 -0
  180. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +92 -0
  181. airbyte_cdk/sources/declarative/schema/schema_loader.py +17 -0
  182. airbyte_cdk/sources/declarative/spec/__init__.py +7 -0
  183. airbyte_cdk/sources/declarative/spec/spec.py +48 -0
  184. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +7 -0
  185. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +93 -0
  186. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +25 -0
  187. airbyte_cdk/sources/declarative/transformations/__init__.py +17 -0
  188. airbyte_cdk/sources/declarative/transformations/add_fields.py +146 -0
  189. airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py +61 -0
  190. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  191. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  192. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  193. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  194. airbyte_cdk/sources/declarative/transformations/remove_fields.py +75 -0
  195. airbyte_cdk/sources/declarative/transformations/transformation.py +37 -0
  196. airbyte_cdk/sources/declarative/types.py +25 -0
  197. airbyte_cdk/sources/declarative/yaml_declarative_source.py +67 -0
  198. airbyte_cdk/sources/file_based/README.md +152 -0
  199. airbyte_cdk/sources/file_based/__init__.py +24 -0
  200. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +11 -0
  201. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +73 -0
  202. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +149 -0
  203. airbyte_cdk/sources/file_based/config/__init__.py +0 -0
  204. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +153 -0
  205. airbyte_cdk/sources/file_based/config/avro_format.py +25 -0
  206. airbyte_cdk/sources/file_based/config/csv_format.py +210 -0
  207. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  208. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +99 -0
  209. airbyte_cdk/sources/file_based/config/jsonl_format.py +18 -0
  210. airbyte_cdk/sources/file_based/config/parquet_format.py +25 -0
  211. airbyte_cdk/sources/file_based/config/unstructured_format.py +102 -0
  212. airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py +81 -0
  213. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +8 -0
  214. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +21 -0
  215. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +33 -0
  216. airbyte_cdk/sources/file_based/exceptions.py +159 -0
  217. airbyte_cdk/sources/file_based/file_based_source.py +466 -0
  218. airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py +123 -0
  219. airbyte_cdk/sources/file_based/file_based_stream_reader.py +209 -0
  220. airbyte_cdk/sources/file_based/file_record_data.py +22 -0
  221. airbyte_cdk/sources/file_based/file_types/__init__.py +37 -0
  222. airbyte_cdk/sources/file_based/file_types/avro_parser.py +233 -0
  223. airbyte_cdk/sources/file_based/file_types/csv_parser.py +527 -0
  224. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  225. airbyte_cdk/sources/file_based/file_types/file_transfer.py +30 -0
  226. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +86 -0
  227. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +145 -0
  228. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +275 -0
  229. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +480 -0
  230. airbyte_cdk/sources/file_based/remote_file.py +18 -0
  231. airbyte_cdk/sources/file_based/schema_helpers.py +281 -0
  232. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +17 -0
  233. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +20 -0
  234. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +52 -0
  235. airbyte_cdk/sources/file_based/stream/__init__.py +13 -0
  236. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +197 -0
  237. airbyte_cdk/sources/file_based/stream/concurrent/__init__.py +0 -0
  238. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +343 -0
  239. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +9 -0
  240. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +59 -0
  241. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +313 -0
  242. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +83 -0
  243. airbyte_cdk/sources/file_based/stream/cursor/__init__.py +4 -0
  244. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +66 -0
  245. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +149 -0
  246. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +396 -0
  247. airbyte_cdk/sources/file_based/stream/identities_stream.py +49 -0
  248. airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +92 -0
  249. airbyte_cdk/sources/file_based/types.py +10 -0
  250. airbyte_cdk/sources/http_config.py +10 -0
  251. airbyte_cdk/sources/http_logger.py +55 -0
  252. airbyte_cdk/sources/message/__init__.py +19 -0
  253. airbyte_cdk/sources/message/repository.py +137 -0
  254. airbyte_cdk/sources/source.py +95 -0
  255. airbyte_cdk/sources/specs/transfer_modes.py +26 -0
  256. airbyte_cdk/sources/streams/__init__.py +8 -0
  257. airbyte_cdk/sources/streams/availability_strategy.py +84 -0
  258. airbyte_cdk/sources/streams/call_rate.py +704 -0
  259. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  260. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  261. airbyte_cdk/sources/streams/checkpoint/cursor.py +77 -0
  262. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  263. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  264. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  265. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  266. airbyte_cdk/sources/streams/concurrent/__init__.py +3 -0
  267. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +96 -0
  268. airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py +37 -0
  269. airbyte_cdk/sources/streams/concurrent/adapters.py +397 -0
  270. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +94 -0
  271. airbyte_cdk/sources/streams/concurrent/clamping.py +99 -0
  272. airbyte_cdk/sources/streams/concurrent/cursor.py +481 -0
  273. airbyte_cdk/sources/streams/concurrent/cursor_types.py +32 -0
  274. airbyte_cdk/sources/streams/concurrent/default_stream.py +102 -0
  275. airbyte_cdk/sources/streams/concurrent/exceptions.py +18 -0
  276. airbyte_cdk/sources/streams/concurrent/helpers.py +42 -0
  277. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +64 -0
  278. airbyte_cdk/sources/streams/concurrent/partition_reader.py +45 -0
  279. airbyte_cdk/sources/streams/concurrent/partitions/__init__.py +3 -0
  280. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +48 -0
  281. airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py +18 -0
  282. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  283. airbyte_cdk/sources/streams/concurrent/partitions/types.py +38 -0
  284. airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py +0 -0
  285. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +182 -0
  286. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +223 -0
  287. airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
  288. airbyte_cdk/sources/streams/core.py +703 -0
  289. airbyte_cdk/sources/streams/http/__init__.py +10 -0
  290. airbyte_cdk/sources/streams/http/availability_strategy.py +54 -0
  291. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  292. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  293. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  294. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  295. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  296. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  297. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  298. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  299. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  300. airbyte_cdk/sources/streams/http/exceptions.py +61 -0
  301. airbyte_cdk/sources/streams/http/http.py +673 -0
  302. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  303. airbyte_cdk/sources/streams/http/rate_limiting.py +158 -0
  304. airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py +14 -0
  305. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +479 -0
  306. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +34 -0
  307. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +436 -0
  308. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +83 -0
  309. airbyte_cdk/sources/streams/permissions/identities_stream.py +75 -0
  310. airbyte_cdk/sources/streams/utils/__init__.py +3 -0
  311. airbyte_cdk/sources/types.py +169 -0
  312. airbyte_cdk/sources/utils/__init__.py +7 -0
  313. airbyte_cdk/sources/utils/casing.py +12 -0
  314. airbyte_cdk/sources/utils/files_directory.py +15 -0
  315. airbyte_cdk/sources/utils/record_helper.py +53 -0
  316. airbyte_cdk/sources/utils/schema_helpers.py +230 -0
  317. airbyte_cdk/sources/utils/slice_logger.py +57 -0
  318. airbyte_cdk/sources/utils/transform.py +277 -0
  319. airbyte_cdk/sources/utils/types.py +7 -0
  320. airbyte_cdk/sql/__init__.py +0 -0
  321. airbyte_cdk/sql/_util/__init__.py +0 -0
  322. airbyte_cdk/sql/_util/hashing.py +34 -0
  323. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  324. airbyte_cdk/sql/constants.py +32 -0
  325. airbyte_cdk/sql/exceptions.py +235 -0
  326. airbyte_cdk/sql/secrets.py +123 -0
  327. airbyte_cdk/sql/shared/__init__.py +15 -0
  328. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  329. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  330. airbyte_cdk/sql/types.py +160 -0
  331. airbyte_cdk/test/__init__.py +7 -0
  332. airbyte_cdk/test/catalog_builder.py +81 -0
  333. airbyte_cdk/test/entrypoint_wrapper.py +250 -0
  334. airbyte_cdk/test/mock_http/__init__.py +6 -0
  335. airbyte_cdk/test/mock_http/matcher.py +41 -0
  336. airbyte_cdk/test/mock_http/mocker.py +185 -0
  337. airbyte_cdk/test/mock_http/request.py +103 -0
  338. airbyte_cdk/test/mock_http/response.py +28 -0
  339. airbyte_cdk/test/mock_http/response_builder.py +237 -0
  340. airbyte_cdk/test/state_builder.py +33 -0
  341. airbyte_cdk/test/utils/__init__.py +1 -0
  342. airbyte_cdk/test/utils/data.py +24 -0
  343. airbyte_cdk/test/utils/http_mocking.py +16 -0
  344. airbyte_cdk/test/utils/manifest_only_fixtures.py +59 -0
  345. airbyte_cdk/test/utils/reading.py +26 -0
  346. airbyte_cdk/utils/__init__.py +10 -0
  347. airbyte_cdk/utils/airbyte_secrets_utils.py +80 -0
  348. airbyte_cdk/utils/analytics_message.py +25 -0
  349. airbyte_cdk/utils/constants.py +5 -0
  350. airbyte_cdk/utils/datetime_format_inferrer.py +94 -0
  351. airbyte_cdk/utils/datetime_helpers.py +499 -0
  352. airbyte_cdk/utils/event_timing.py +85 -0
  353. airbyte_cdk/utils/is_cloud_environment.py +18 -0
  354. airbyte_cdk/utils/mapping_helpers.py +162 -0
  355. airbyte_cdk/utils/message_utils.py +26 -0
  356. airbyte_cdk/utils/oneof_option_config.py +33 -0
  357. airbyte_cdk/utils/print_buffer.py +75 -0
  358. airbyte_cdk/utils/schema_inferrer.py +270 -0
  359. airbyte_cdk/utils/slice_hasher.py +37 -0
  360. airbyte_cdk/utils/spec_schema_transformations.py +26 -0
  361. airbyte_cdk/utils/stream_status_utils.py +43 -0
  362. airbyte_cdk/utils/traced_exception.py +145 -0
  363. airbyte_cdk-0.0.0.dev0.dist-info/LICENSE.txt +19 -0
  364. airbyte_cdk-0.0.0.dev0.dist-info/LICENSE_SHORT +1 -0
  365. airbyte_cdk-0.0.0.dev0.dist-info/METADATA +111 -0
  366. airbyte_cdk-0.0.0.dev0.dist-info/RECORD +368 -0
  367. airbyte_cdk-0.0.0.dev0.dist-info/WHEEL +4 -0
  368. airbyte_cdk-0.0.0.dev0.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,39 @@
1
+ # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
+ from datetime import datetime, timedelta, timezone
3
+ from typing import Optional
4
+
5
+
6
+ class Timer:
7
+ def __init__(self, timeout: timedelta) -> None:
8
+ self._start_datetime: Optional[datetime] = None
9
+ self._end_datetime: Optional[datetime] = None
10
+ self._timeout = timeout
11
+
12
+ def start(self) -> None:
13
+ self._start_datetime = self._now()
14
+ self._end_datetime = None
15
+
16
+ def stop(self) -> None:
17
+ if self._end_datetime is None:
18
+ self._end_datetime = self._now()
19
+
20
+ def is_started(self) -> bool:
21
+ return self._start_datetime is not None
22
+
23
+ @property
24
+ def elapsed_time(self) -> Optional[timedelta]:
25
+ if not self._start_datetime:
26
+ return None
27
+
28
+ end_time = self._end_datetime or self._now()
29
+ elapsed_period = end_time - self._start_datetime
30
+ return elapsed_period
31
+
32
+ def has_timed_out(self) -> bool:
33
+ if not self.is_started():
34
+ return False
35
+ return self.elapsed_time > self._timeout # type: ignore # given the job timer is started, we assume there is an elapsed_period
36
+
37
+ @staticmethod
38
+ def _now() -> datetime:
39
+ return datetime.now(tz=timezone.utc)
@@ -0,0 +1,8 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from airbyte_cdk.sources.declarative.auth.jwt import JwtAuthenticator
6
+ from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeOauth2Authenticator
7
+
8
+ __all__ = ["DeclarativeOauth2Authenticator", "JwtAuthenticator"]
@@ -0,0 +1,42 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from dataclasses import InitVar, dataclass
6
+ from typing import Any, Mapping, Union
7
+
8
+ from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import (
9
+ AbstractHeaderAuthenticator,
10
+ )
11
+
12
+
13
+ @dataclass
14
+ class DeclarativeAuthenticator(AbstractHeaderAuthenticator):
15
+ """
16
+ Interface used to associate which authenticators can be used as part of the declarative framework
17
+ """
18
+
19
+ def get_request_params(self) -> Mapping[str, Any]:
20
+ """HTTP request parameter to add to the requests"""
21
+ return {}
22
+
23
+ def get_request_body_data(self) -> Union[Mapping[str, Any], str]:
24
+ """Form-encoded body data to set on the requests"""
25
+ return {}
26
+
27
+ def get_request_body_json(self) -> Mapping[str, Any]:
28
+ """JSON-encoded body data to set on the requests"""
29
+ return {}
30
+
31
+
32
+ @dataclass
33
+ class NoAuth(DeclarativeAuthenticator):
34
+ parameters: InitVar[Mapping[str, Any]]
35
+
36
+ @property
37
+ def auth_header(self) -> str:
38
+ return ""
39
+
40
+ @property
41
+ def token(self) -> str:
42
+ return ""
@@ -0,0 +1,197 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ import base64
6
+ import json
7
+ from dataclasses import InitVar, dataclass
8
+ from datetime import datetime
9
+ from typing import Any, Mapping, Optional, Union
10
+
11
+ import jwt
12
+
13
+ from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
14
+ from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
15
+ from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
16
+ from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
17
+
18
+
19
+ class JwtAlgorithm(str):
20
+ """
21
+ Enum for supported JWT algorithms
22
+ """
23
+
24
+ HS256 = "HS256"
25
+ HS384 = "HS384"
26
+ HS512 = "HS512"
27
+ ES256 = "ES256"
28
+ ES256K = "ES256K"
29
+ ES384 = "ES384"
30
+ ES512 = "ES512"
31
+ RS256 = "RS256"
32
+ RS384 = "RS384"
33
+ RS512 = "RS512"
34
+ PS256 = "PS256"
35
+ PS384 = "PS384"
36
+ PS512 = "PS512"
37
+ EdDSA = "EdDSA"
38
+
39
+
40
+ @dataclass
41
+ class JwtAuthenticator(DeclarativeAuthenticator):
42
+ """
43
+ Generates a JSON Web Token (JWT) based on a declarative connector configuration file. The generated token is attached to each request via the Authorization header.
44
+
45
+ Attributes:
46
+ config (Mapping[str, Any]): The user-provided configuration as specified by the source's spec
47
+ secret_key (Union[InterpolatedString, str]): The secret key used to sign the JWT
48
+ algorithm (Union[str, JwtAlgorithm]): The algorithm used to sign the JWT
49
+ token_duration (Optional[int]): The duration in seconds for which the token is valid
50
+ base64_encode_secret_key (Optional[Union[InterpolatedBoolean, str, bool]]): Whether to base64 encode the secret key
51
+ header_prefix (Optional[Union[InterpolatedString, str]]): The prefix to add to the Authorization header
52
+ kid (Optional[Union[InterpolatedString, str]]): The key identifier to be included in the JWT header
53
+ typ (Optional[Union[InterpolatedString, str]]): The type of the JWT.
54
+ cty (Optional[Union[InterpolatedString, str]]): The content type of the JWT.
55
+ iss (Optional[Union[InterpolatedString, str]]): The issuer of the JWT.
56
+ sub (Optional[Union[InterpolatedString, str]]): The subject of the JWT.
57
+ aud (Optional[Union[InterpolatedString, str]]): The audience of the JWT.
58
+ additional_jwt_headers (Optional[Mapping[str, Any]]): Additional headers to include in the JWT.
59
+ additional_jwt_payload (Optional[Mapping[str, Any]]): Additional payload to include in the JWT.
60
+ """
61
+
62
+ config: Mapping[str, Any]
63
+ parameters: InitVar[Mapping[str, Any]]
64
+ secret_key: Union[InterpolatedString, str]
65
+ algorithm: Union[str, JwtAlgorithm]
66
+ token_duration: Optional[int]
67
+ base64_encode_secret_key: Optional[Union[InterpolatedBoolean, str, bool]] = False
68
+ header_prefix: Optional[Union[InterpolatedString, str]] = None
69
+ kid: Optional[Union[InterpolatedString, str]] = None
70
+ typ: Optional[Union[InterpolatedString, str]] = None
71
+ cty: Optional[Union[InterpolatedString, str]] = None
72
+ iss: Optional[Union[InterpolatedString, str]] = None
73
+ sub: Optional[Union[InterpolatedString, str]] = None
74
+ aud: Optional[Union[InterpolatedString, str]] = None
75
+ additional_jwt_headers: Optional[Mapping[str, Any]] = None
76
+ additional_jwt_payload: Optional[Mapping[str, Any]] = None
77
+
78
+ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
79
+ self._secret_key = InterpolatedString.create(self.secret_key, parameters=parameters)
80
+ self._algorithm = (
81
+ JwtAlgorithm(self.algorithm) if isinstance(self.algorithm, str) else self.algorithm
82
+ )
83
+ self._base64_encode_secret_key = (
84
+ InterpolatedBoolean(self.base64_encode_secret_key, parameters=parameters)
85
+ if isinstance(self.base64_encode_secret_key, str)
86
+ else self.base64_encode_secret_key
87
+ )
88
+ self._token_duration = self.token_duration
89
+ self._header_prefix = (
90
+ InterpolatedString.create(self.header_prefix, parameters=parameters)
91
+ if self.header_prefix
92
+ else None
93
+ )
94
+ self._kid = InterpolatedString.create(self.kid, parameters=parameters) if self.kid else None
95
+ self._typ = InterpolatedString.create(self.typ, parameters=parameters) if self.typ else None
96
+ self._cty = InterpolatedString.create(self.cty, parameters=parameters) if self.cty else None
97
+ self._iss = InterpolatedString.create(self.iss, parameters=parameters) if self.iss else None
98
+ self._sub = InterpolatedString.create(self.sub, parameters=parameters) if self.sub else None
99
+ self._aud = InterpolatedString.create(self.aud, parameters=parameters) if self.aud else None
100
+ self._additional_jwt_headers = InterpolatedMapping(
101
+ self.additional_jwt_headers or {}, parameters=parameters
102
+ )
103
+ self._additional_jwt_payload = InterpolatedMapping(
104
+ self.additional_jwt_payload or {}, parameters=parameters
105
+ )
106
+
107
+ def _get_jwt_headers(self) -> dict[str, Any]:
108
+ """
109
+ Builds and returns the headers used when signing the JWT.
110
+ """
111
+ headers = self._additional_jwt_headers.eval(self.config, json_loads=json.loads)
112
+ if any(prop in headers for prop in ["kid", "alg", "typ", "cty"]):
113
+ raise ValueError(
114
+ "'kid', 'alg', 'typ', 'cty' are reserved headers and should not be set as part of 'additional_jwt_headers'"
115
+ )
116
+
117
+ if self._kid:
118
+ headers["kid"] = self._kid.eval(self.config, json_loads=json.loads)
119
+ if self._typ:
120
+ headers["typ"] = self._typ.eval(self.config, json_loads=json.loads)
121
+ if self._cty:
122
+ headers["cty"] = self._cty.eval(self.config, json_loads=json.loads)
123
+ headers["alg"] = self._algorithm
124
+ return headers
125
+
126
+ def _get_jwt_payload(self) -> dict[str, Any]:
127
+ """
128
+ Builds and returns the payload used when signing the JWT.
129
+ """
130
+ now = int(datetime.now().timestamp())
131
+ exp = now + self._token_duration if isinstance(self._token_duration, int) else now
132
+ nbf = now
133
+
134
+ payload = self._additional_jwt_payload.eval(self.config, json_loads=json.loads)
135
+ if any(prop in payload for prop in ["iss", "sub", "aud", "iat", "exp", "nbf"]):
136
+ raise ValueError(
137
+ "'iss', 'sub', 'aud', 'iat', 'exp', 'nbf' are reserved properties and should not be set as part of 'additional_jwt_payload'"
138
+ )
139
+
140
+ if self._iss:
141
+ payload["iss"] = self._iss.eval(self.config, json_loads=json.loads)
142
+ if self._sub:
143
+ payload["sub"] = self._sub.eval(self.config, json_loads=json.loads)
144
+ if self._aud:
145
+ payload["aud"] = self._aud.eval(self.config, json_loads=json.loads)
146
+
147
+ payload["iat"] = now
148
+ payload["exp"] = exp
149
+ payload["nbf"] = nbf
150
+ return payload
151
+
152
+ def _get_secret_key(self) -> str:
153
+ """
154
+ Returns the secret key used to sign the JWT.
155
+ """
156
+ secret_key: str = self._secret_key.eval(self.config, json_loads=json.loads)
157
+ return (
158
+ base64.b64encode(secret_key.encode()).decode()
159
+ if self._base64_encode_secret_key
160
+ else secret_key
161
+ )
162
+
163
+ def _get_signed_token(self) -> Union[str, Any]:
164
+ """
165
+ Signed the JWT using the provided secret key and algorithm and the generated headers and payload. For additional information on PyJWT see: https://pyjwt.readthedocs.io/en/stable/
166
+ """
167
+ try:
168
+ return jwt.encode(
169
+ payload=self._get_jwt_payload(),
170
+ key=self._get_secret_key(),
171
+ algorithm=self._algorithm,
172
+ headers=self._get_jwt_headers(),
173
+ )
174
+ except Exception as e:
175
+ raise ValueError(f"Failed to sign token: {e}")
176
+
177
+ def _get_header_prefix(self) -> Union[str, None]:
178
+ """
179
+ Returns the header prefix to be used when attaching the token to the request.
180
+ """
181
+ return (
182
+ self._header_prefix.eval(self.config, json_loads=json.loads)
183
+ if self._header_prefix
184
+ else None
185
+ )
186
+
187
+ @property
188
+ def auth_header(self) -> str:
189
+ return "Authorization"
190
+
191
+ @property
192
+ def token(self) -> str:
193
+ return (
194
+ f"{self._get_header_prefix()} {self._get_signed_token()}"
195
+ if self._get_header_prefix()
196
+ else self._get_signed_token()
197
+ )
@@ -0,0 +1,293 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from dataclasses import InitVar, dataclass, field
6
+ from datetime import datetime, timedelta
7
+ from typing import Any, List, Mapping, MutableMapping, Optional, Union
8
+
9
+ from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
10
+ from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
11
+ from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
12
+ from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
13
+ from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
14
+ from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import (
15
+ AbstractOauth2Authenticator,
16
+ )
17
+ from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import (
18
+ SingleUseRefreshTokenOauth2Authenticator,
19
+ )
20
+ from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
21
+
22
+
23
+ @dataclass
24
+ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAuthenticator):
25
+ """
26
+ Generates OAuth2.0 access tokens from an OAuth2.0 refresh token and client credentials based on
27
+ a declarative connector configuration file. Credentials can be defined explicitly or via interpolation
28
+ at runtime. The generated access token is attached to each request via the Authorization header.
29
+
30
+ Attributes:
31
+ token_refresh_endpoint (Union[InterpolatedString, str]): The endpoint to refresh the access token
32
+ client_id (Union[InterpolatedString, str]): The client id
33
+ client_secret (Union[InterpolatedString, str]): Client secret
34
+ refresh_token (Union[InterpolatedString, str]): The token used to refresh the access token
35
+ access_token_name (Union[InterpolatedString, str]): THe field to extract access token from in the response
36
+ expires_in_name (Union[InterpolatedString, str]): The field to extract expires_in from in the response
37
+ config (Mapping[str, Any]): The user-provided configuration as specified by the source's spec
38
+ scopes (Optional[List[str]]): The scopes to request
39
+ token_expiry_date (Optional[Union[InterpolatedString, str]]): The access token expiration date
40
+ token_expiry_date_format str: format of the datetime; provide it if expires_in is returned in datetime instead of seconds
41
+ token_expiry_is_time_of_expiration bool: set True it if expires_in is returned as time of expiration instead of the number seconds until expiration
42
+ refresh_request_body (Optional[Mapping[str, Any]]): The request body to send in the refresh request
43
+ refresh_request_headers (Optional[Mapping[str, Any]]): The request headers to send in the refresh request
44
+ grant_type: The grant_type to request for access_token. If set to refresh_token, the refresh_token parameter has to be provided
45
+ message_repository (MessageRepository): the message repository used to emit logs on HTTP requests
46
+ """
47
+
48
+ config: Mapping[str, Any]
49
+ parameters: InitVar[Mapping[str, Any]]
50
+ client_id: Optional[Union[InterpolatedString, str]] = None
51
+ client_secret: Optional[Union[InterpolatedString, str]] = None
52
+ token_refresh_endpoint: Optional[Union[InterpolatedString, str]] = None
53
+ refresh_token: Optional[Union[InterpolatedString, str]] = None
54
+ scopes: Optional[List[str]] = None
55
+ token_expiry_date: Optional[Union[InterpolatedString, str]] = None
56
+ _token_expiry_date: Optional[AirbyteDateTime] = field(init=False, repr=False, default=None)
57
+ token_expiry_date_format: Optional[str] = None
58
+ token_expiry_is_time_of_expiration: bool = False
59
+ access_token_name: Union[InterpolatedString, str] = "access_token"
60
+ access_token_value: Optional[Union[InterpolatedString, str]] = None
61
+ client_id_name: Union[InterpolatedString, str] = "client_id"
62
+ client_secret_name: Union[InterpolatedString, str] = "client_secret"
63
+ expires_in_name: Union[InterpolatedString, str] = "expires_in"
64
+ refresh_token_name: Union[InterpolatedString, str] = "refresh_token"
65
+ refresh_request_body: Optional[Mapping[str, Any]] = None
66
+ refresh_request_headers: Optional[Mapping[str, Any]] = None
67
+ grant_type_name: Union[InterpolatedString, str] = "grant_type"
68
+ grant_type: Union[InterpolatedString, str] = "refresh_token"
69
+ message_repository: MessageRepository = NoopMessageRepository()
70
+ profile_assertion: Optional[DeclarativeAuthenticator] = None
71
+ use_profile_assertion: Optional[Union[InterpolatedBoolean, str, bool]] = False
72
+
73
+ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
74
+ super().__init__()
75
+ if self.token_refresh_endpoint is not None:
76
+ self._token_refresh_endpoint: Optional[InterpolatedString] = InterpolatedString.create(
77
+ self.token_refresh_endpoint, parameters=parameters
78
+ )
79
+ else:
80
+ self._token_refresh_endpoint = None
81
+ self._client_id_name = InterpolatedString.create(self.client_id_name, parameters=parameters)
82
+ self._client_id = (
83
+ InterpolatedString.create(self.client_id, parameters=parameters)
84
+ if self.client_id
85
+ else self.client_id
86
+ )
87
+ self._client_secret_name = InterpolatedString.create(
88
+ self.client_secret_name, parameters=parameters
89
+ )
90
+ self._client_secret = (
91
+ InterpolatedString.create(self.client_secret, parameters=parameters)
92
+ if self.client_secret
93
+ else self.client_secret
94
+ )
95
+ self._refresh_token_name = InterpolatedString.create(
96
+ self.refresh_token_name, parameters=parameters
97
+ )
98
+ if self.refresh_token is not None:
99
+ self._refresh_token: Optional[InterpolatedString] = InterpolatedString.create(
100
+ self.refresh_token, parameters=parameters
101
+ )
102
+ else:
103
+ self._refresh_token = None
104
+ self.access_token_name = InterpolatedString.create(
105
+ self.access_token_name, parameters=parameters
106
+ )
107
+ self.expires_in_name = InterpolatedString.create(
108
+ self.expires_in_name, parameters=parameters
109
+ )
110
+ self.grant_type_name = InterpolatedString.create(
111
+ self.grant_type_name, parameters=parameters
112
+ )
113
+ self.grant_type = InterpolatedString.create(
114
+ "urn:ietf:params:oauth:grant-type:jwt-bearer"
115
+ if self.use_profile_assertion
116
+ else self.grant_type,
117
+ parameters=parameters,
118
+ )
119
+ self._refresh_request_body = InterpolatedMapping(
120
+ self.refresh_request_body or {}, parameters=parameters
121
+ )
122
+ self._refresh_request_headers = InterpolatedMapping(
123
+ self.refresh_request_headers or {}, parameters=parameters
124
+ )
125
+ try:
126
+ if (
127
+ isinstance(self.token_expiry_date, (int, str))
128
+ and str(self.token_expiry_date).isdigit()
129
+ ):
130
+ self._token_expiry_date = ab_datetime_parse(self.token_expiry_date)
131
+ else:
132
+ self._token_expiry_date = (
133
+ ab_datetime_parse(
134
+ InterpolatedString.create(
135
+ self.token_expiry_date, parameters=parameters
136
+ ).eval(self.config)
137
+ )
138
+ if self.token_expiry_date
139
+ else ab_datetime_now() - timedelta(days=1)
140
+ )
141
+ except ValueError as e:
142
+ raise ValueError(f"Invalid token expiry date format: {e}")
143
+ self.use_profile_assertion = (
144
+ InterpolatedBoolean(self.use_profile_assertion, parameters=parameters)
145
+ if isinstance(self.use_profile_assertion, str)
146
+ else self.use_profile_assertion
147
+ )
148
+ self.assertion_name = "assertion"
149
+
150
+ if self.access_token_value is not None:
151
+ self._access_token_value = InterpolatedString.create(
152
+ self.access_token_value, parameters=parameters
153
+ ).eval(self.config)
154
+ else:
155
+ self._access_token_value = None
156
+
157
+ self._access_token: Optional[str] = (
158
+ self._access_token_value if self.access_token_value else None
159
+ )
160
+
161
+ if not self.use_profile_assertion and any(
162
+ client_creds is None for client_creds in [self.client_id, self.client_secret]
163
+ ):
164
+ raise ValueError(
165
+ "OAuthAuthenticator configuration error: Both 'client_id' and 'client_secret' are required for the "
166
+ "basic OAuth flow."
167
+ )
168
+ if self.profile_assertion is None and self.use_profile_assertion:
169
+ raise ValueError(
170
+ "OAuthAuthenticator configuration error: 'profile_assertion' is required when using the profile assertion flow."
171
+ )
172
+ if self.get_grant_type() == "refresh_token" and self._refresh_token is None:
173
+ raise ValueError(
174
+ "OAuthAuthenticator configuration error: A 'refresh_token' is required when the 'grant_type' is set to 'refresh_token'."
175
+ )
176
+
177
+ def get_token_refresh_endpoint(self) -> Optional[str]:
178
+ if self._token_refresh_endpoint is not None:
179
+ refresh_token_endpoint: str = self._token_refresh_endpoint.eval(self.config)
180
+ if not refresh_token_endpoint:
181
+ raise ValueError(
182
+ "OAuthAuthenticator was unable to evaluate token_refresh_endpoint parameter"
183
+ )
184
+ return refresh_token_endpoint
185
+ return None
186
+
187
+ def get_client_id_name(self) -> str:
188
+ return self._client_id_name.eval(self.config) # type: ignore # eval returns a string in this context
189
+
190
+ def get_client_id(self) -> str:
191
+ client_id = self._client_id.eval(self.config) if self._client_id else self._client_id
192
+ if not client_id:
193
+ raise ValueError("OAuthAuthenticator was unable to evaluate client_id parameter")
194
+ return client_id # type: ignore # value will be returned as a string, or an error will be raised
195
+
196
+ def get_client_secret_name(self) -> str:
197
+ return self._client_secret_name.eval(self.config) # type: ignore # eval returns a string in this context
198
+
199
+ def get_client_secret(self) -> str:
200
+ client_secret = (
201
+ self._client_secret.eval(self.config) if self._client_secret else self._client_secret
202
+ )
203
+ if not client_secret:
204
+ raise ValueError("OAuthAuthenticator was unable to evaluate client_secret parameter")
205
+ return client_secret # type: ignore # value will be returned as a string, or an error will be raised
206
+
207
+ def get_refresh_token_name(self) -> str:
208
+ return self._refresh_token_name.eval(self.config) # type: ignore # eval returns a string in this context
209
+
210
+ def get_refresh_token(self) -> Optional[str]:
211
+ return None if self._refresh_token is None else str(self._refresh_token.eval(self.config))
212
+
213
+ def get_scopes(self) -> List[str]:
214
+ return self.scopes or []
215
+
216
+ def get_access_token_name(self) -> str:
217
+ return self.access_token_name.eval(self.config) # type: ignore # eval returns a string in this context
218
+
219
+ def get_expires_in_name(self) -> str:
220
+ return self.expires_in_name.eval(self.config) # type: ignore # eval returns a string in this context
221
+
222
+ def get_grant_type_name(self) -> str:
223
+ return self.grant_type_name.eval(self.config) # type: ignore # eval returns a string in this context
224
+
225
+ def get_grant_type(self) -> str:
226
+ return self.grant_type.eval(self.config) # type: ignore # eval returns a string in this context
227
+
228
+ def get_refresh_request_body(self) -> Mapping[str, Any]:
229
+ return self._refresh_request_body.eval(self.config)
230
+
231
+ def get_refresh_request_headers(self) -> Mapping[str, Any]:
232
+ return self._refresh_request_headers.eval(self.config)
233
+
234
+ def get_token_expiry_date(self) -> AirbyteDateTime:
235
+ if not self._has_access_token_been_initialized():
236
+ return AirbyteDateTime.from_datetime(datetime.min)
237
+ return self._token_expiry_date # type: ignore # _token_expiry_date is an AirbyteDateTime. It is never None despite what mypy thinks
238
+
239
+ def _has_access_token_been_initialized(self) -> bool:
240
+ return self._access_token is not None
241
+
242
+ def set_token_expiry_date(self, value: Union[str, int]) -> None:
243
+ self._token_expiry_date = self._parse_token_expiration_date(value)
244
+
245
+ def get_assertion_name(self) -> str:
246
+ return self.assertion_name
247
+
248
+ def get_assertion(self) -> str:
249
+ if self.profile_assertion is None:
250
+ raise ValueError("profile_assertion is not set")
251
+ return self.profile_assertion.token
252
+
253
+ def build_refresh_request_body(self) -> Mapping[str, Any]:
254
+ """
255
+ Returns the request body to set on the refresh request
256
+
257
+ Override to define additional parameters
258
+ """
259
+ if self.use_profile_assertion:
260
+ return {
261
+ self.get_grant_type_name(): self.get_grant_type(),
262
+ self.get_assertion_name(): self.get_assertion(),
263
+ }
264
+ return super().build_refresh_request_body()
265
+
266
+ @property
267
+ def access_token(self) -> str:
268
+ if self._access_token is None:
269
+ raise ValueError("access_token is not set")
270
+ return self._access_token
271
+
272
+ @access_token.setter
273
+ def access_token(self, value: str) -> None:
274
+ self._access_token = value
275
+
276
+ @property
277
+ def _message_repository(self) -> MessageRepository:
278
+ """
279
+ Overriding AbstractOauth2Authenticator._message_repository to allow for HTTP request logs
280
+ """
281
+ return self.message_repository
282
+
283
+
284
+ @dataclass
285
+ class DeclarativeSingleUseRefreshTokenOauth2Authenticator(
286
+ SingleUseRefreshTokenOauth2Authenticator, DeclarativeAuthenticator
287
+ ):
288
+ """
289
+ Declarative version of SingleUseRefreshTokenOauth2Authenticator which can be used in declarative connectors.
290
+ """
291
+
292
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
293
+ super().__init__(*args, **kwargs)
@@ -0,0 +1,45 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from dataclasses import dataclass
6
+ from typing import Any, List, Mapping
7
+
8
+ import dpath
9
+
10
+ from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
11
+
12
+
13
+ @dataclass
14
+ class SelectiveAuthenticator(DeclarativeAuthenticator):
15
+ """Authenticator that selects concrete implementation based on specific config value."""
16
+
17
+ config: Mapping[str, Any]
18
+ authenticators: Mapping[str, DeclarativeAuthenticator]
19
+ authenticator_selection_path: List[str]
20
+
21
+ # returns "DeclarativeAuthenticator", but must return a subtype of "SelectiveAuthenticator"
22
+ def __new__( # type: ignore[misc]
23
+ cls,
24
+ config: Mapping[str, Any],
25
+ authenticators: Mapping[str, DeclarativeAuthenticator],
26
+ authenticator_selection_path: List[str],
27
+ *arg: Any,
28
+ **kwargs: Any,
29
+ ) -> DeclarativeAuthenticator:
30
+ try:
31
+ selected_key = str(
32
+ dpath.get(
33
+ config, # type: ignore[arg-type] # Dpath wants mutable mapping but doesn't need it.
34
+ authenticator_selection_path,
35
+ )
36
+ )
37
+ except KeyError as err:
38
+ raise ValueError(
39
+ "The path from `authenticator_selection_path` is not found in the config."
40
+ ) from err
41
+
42
+ try:
43
+ return authenticators[selected_key]
44
+ except KeyError as err:
45
+ raise ValueError(f"The authenticator `{selected_key}` is not found.") from err