airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (518) hide show
  1. airbyte_cdk/__init__.py +355 -6
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +29 -10
  7. airbyte_cdk/connector.py +24 -24
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
  10. airbyte_cdk/connector_builder/main.py +45 -13
  11. airbyte_cdk/connector_builder/message_grouper.py +189 -50
  12. airbyte_cdk/connector_builder/models.py +3 -2
  13. airbyte_cdk/destinations/__init__.py +4 -3
  14. airbyte_cdk/destinations/destination.py +54 -20
  15. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  16. airbyte_cdk/destinations/vector_db_based/config.py +40 -17
  17. airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
  18. airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
  19. airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
  20. airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
  21. airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
  22. airbyte_cdk/entrypoint.py +153 -44
  23. airbyte_cdk/exception_handler.py +21 -3
  24. airbyte_cdk/logger.py +30 -44
  25. airbyte_cdk/models/__init__.py +13 -2
  26. airbyte_cdk/models/airbyte_protocol.py +86 -1
  27. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  28. airbyte_cdk/models/file_transfer_record_message.py +13 -0
  29. airbyte_cdk/models/well_known_types.py +1 -1
  30. airbyte_cdk/sources/__init__.py +5 -1
  31. airbyte_cdk/sources/abstract_source.py +125 -79
  32. airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
  33. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
  34. airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
  35. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
  36. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  37. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
  38. airbyte_cdk/sources/config.py +3 -2
  39. airbyte_cdk/sources/connector_state_manager.py +49 -83
  40. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  41. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
  42. airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
  43. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  44. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  45. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  46. airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
  47. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
  48. airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
  49. airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
  50. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
  51. airbyte_cdk/sources/declarative/auth/token.py +28 -10
  52. airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
  53. airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
  54. airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
  55. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  56. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  57. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
  58. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
  59. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
  60. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1213 -88
  61. airbyte_cdk/sources/declarative/declarative_source.py +5 -2
  62. airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
  63. airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
  64. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
  65. airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
  66. airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
  67. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  68. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  69. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  70. airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
  71. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
  72. airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
  73. airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
  74. airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
  75. airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
  76. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
  77. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  78. airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
  79. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +346 -0
  80. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
  81. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  82. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
  83. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +173 -74
  84. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  85. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  86. airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
  87. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
  88. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
  89. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
  90. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
  91. airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
  92. airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
  93. airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
  94. airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
  95. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  96. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  97. airbyte_cdk/sources/declarative/models/__init__.py +1 -1
  98. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1329 -595
  99. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
  100. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
  101. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
  102. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1763 -226
  103. airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
  104. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  105. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  106. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
  107. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  108. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
  109. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
  110. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
  111. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
  112. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
  113. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
  114. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
  115. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
  116. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
  117. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
  118. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
  119. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
  120. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  121. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
  122. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
  123. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
  124. airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
  125. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
  126. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
  127. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
  128. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
  129. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
  130. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
  131. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
  132. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
  133. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
  134. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
  135. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
  136. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
  137. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  138. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
  139. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
  140. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
  141. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
  142. airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
  143. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  144. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  145. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  146. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  147. airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
  148. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
  149. airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
  150. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
  151. airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
  152. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
  153. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
  154. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
  155. airbyte_cdk/sources/declarative/spec/spec.py +12 -5
  156. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
  157. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
  158. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
  159. airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
  160. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  161. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  162. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  163. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  164. airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
  165. airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
  166. airbyte_cdk/sources/declarative/types.py +19 -110
  167. airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
  168. airbyte_cdk/sources/embedded/base_integration.py +16 -5
  169. airbyte_cdk/sources/embedded/catalog.py +16 -4
  170. airbyte_cdk/sources/embedded/runner.py +19 -3
  171. airbyte_cdk/sources/embedded/tools.py +5 -2
  172. airbyte_cdk/sources/file_based/README.md +152 -0
  173. airbyte_cdk/sources/file_based/__init__.py +24 -0
  174. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
  175. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
  176. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
  177. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
  178. airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
  179. airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
  180. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  181. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
  182. airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
  183. airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
  184. airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
  185. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
  186. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
  187. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
  188. airbyte_cdk/sources/file_based/exceptions.py +18 -15
  189. airbyte_cdk/sources/file_based/file_based_source.py +140 -33
  190. airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
  191. airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
  192. airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
  193. airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
  194. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  195. airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
  196. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
  197. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
  198. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
  199. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
  200. airbyte_cdk/sources/file_based/remote_file.py +1 -1
  201. airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
  202. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
  203. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
  204. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
  205. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
  206. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
  207. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
  208. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
  209. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
  210. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
  211. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
  212. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
  213. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
  214. airbyte_cdk/sources/http_logger.py +8 -3
  215. airbyte_cdk/sources/message/__init__.py +7 -1
  216. airbyte_cdk/sources/message/repository.py +18 -4
  217. airbyte_cdk/sources/source.py +42 -38
  218. airbyte_cdk/sources/streams/__init__.py +2 -2
  219. airbyte_cdk/sources/streams/availability_strategy.py +54 -3
  220. airbyte_cdk/sources/streams/call_rate.py +64 -21
  221. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  222. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  223. airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
  224. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  225. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  226. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  227. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  228. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
  229. airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
  230. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
  231. airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
  232. airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
  233. airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
  234. airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
  235. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
  236. airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
  237. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
  238. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  239. airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
  240. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
  241. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
  242. airbyte_cdk/sources/streams/core.py +412 -87
  243. airbyte_cdk/sources/streams/http/__init__.py +2 -1
  244. airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
  245. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  246. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  247. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  248. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  249. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  250. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  251. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  252. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  253. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  254. airbyte_cdk/sources/streams/http/exceptions.py +27 -7
  255. airbyte_cdk/sources/streams/http/http.py +369 -246
  256. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  257. airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
  258. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
  259. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
  260. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
  261. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
  262. airbyte_cdk/sources/types.py +154 -0
  263. airbyte_cdk/sources/utils/record_helper.py +36 -21
  264. airbyte_cdk/sources/utils/schema_helpers.py +13 -6
  265. airbyte_cdk/sources/utils/slice_logger.py +4 -1
  266. airbyte_cdk/sources/utils/transform.py +54 -20
  267. airbyte_cdk/sql/_util/hashing.py +34 -0
  268. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  269. airbyte_cdk/sql/constants.py +32 -0
  270. airbyte_cdk/sql/exceptions.py +235 -0
  271. airbyte_cdk/sql/secrets.py +123 -0
  272. airbyte_cdk/sql/shared/__init__.py +15 -0
  273. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  274. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  275. airbyte_cdk/sql/types.py +160 -0
  276. airbyte_cdk/test/catalog_builder.py +70 -18
  277. airbyte_cdk/test/entrypoint_wrapper.py +117 -42
  278. airbyte_cdk/test/mock_http/__init__.py +1 -1
  279. airbyte_cdk/test/mock_http/matcher.py +6 -0
  280. airbyte_cdk/test/mock_http/mocker.py +57 -10
  281. airbyte_cdk/test/mock_http/request.py +19 -3
  282. airbyte_cdk/test/mock_http/response.py +3 -1
  283. airbyte_cdk/test/mock_http/response_builder.py +32 -16
  284. airbyte_cdk/test/state_builder.py +18 -10
  285. airbyte_cdk/test/utils/__init__.py +1 -0
  286. airbyte_cdk/test/utils/data.py +24 -0
  287. airbyte_cdk/test/utils/http_mocking.py +16 -0
  288. airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
  289. airbyte_cdk/test/utils/reading.py +26 -0
  290. airbyte_cdk/utils/__init__.py +2 -1
  291. airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
  292. airbyte_cdk/utils/analytics_message.py +10 -2
  293. airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
  294. airbyte_cdk/utils/event_timing.py +10 -10
  295. airbyte_cdk/utils/mapping_helpers.py +3 -1
  296. airbyte_cdk/utils/message_utils.py +20 -11
  297. airbyte_cdk/utils/print_buffer.py +75 -0
  298. airbyte_cdk/utils/schema_inferrer.py +198 -28
  299. airbyte_cdk/utils/slice_hasher.py +30 -0
  300. airbyte_cdk/utils/spec_schema_transformations.py +6 -3
  301. airbyte_cdk/utils/stream_status_utils.py +8 -1
  302. airbyte_cdk/utils/traced_exception.py +61 -21
  303. airbyte_cdk-6.17.1.dev0.dist-info/METADATA +109 -0
  304. airbyte_cdk-6.17.1.dev0.dist-info/RECORD +350 -0
  305. {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/WHEEL +1 -2
  306. airbyte_cdk-6.17.1.dev0.dist-info/entry_points.txt +3 -0
  307. airbyte_cdk/sources/declarative/create_partial.py +0 -92
  308. airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
  309. airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
  310. airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
  311. airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
  312. airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
  313. airbyte_cdk/sources/deprecated/base_source.py +0 -94
  314. airbyte_cdk/sources/deprecated/client.py +0 -99
  315. airbyte_cdk/sources/singer/__init__.py +0 -8
  316. airbyte_cdk/sources/singer/singer_helpers.py +0 -304
  317. airbyte_cdk/sources/singer/source.py +0 -186
  318. airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
  319. airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
  320. airbyte_cdk/sources/streams/http/auth/core.py +0 -29
  321. airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
  322. airbyte_cdk/sources/streams/http/auth/token.py +0 -47
  323. airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
  324. airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
  325. airbyte_cdk/sources/utils/schema_models.py +0 -84
  326. airbyte_cdk-0.72.0.dist-info/METADATA +0 -243
  327. airbyte_cdk-0.72.0.dist-info/RECORD +0 -466
  328. airbyte_cdk-0.72.0.dist-info/top_level.txt +0 -3
  329. source_declarative_manifest/main.py +0 -29
  330. unit_tests/connector_builder/__init__.py +0 -3
  331. unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
  332. unit_tests/connector_builder/test_message_grouper.py +0 -713
  333. unit_tests/connector_builder/utils.py +0 -27
  334. unit_tests/destinations/test_destination.py +0 -243
  335. unit_tests/singer/test_singer_helpers.py +0 -56
  336. unit_tests/singer/test_singer_source.py +0 -112
  337. unit_tests/sources/__init__.py +0 -0
  338. unit_tests/sources/concurrent_source/__init__.py +0 -3
  339. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
  340. unit_tests/sources/declarative/__init__.py +0 -3
  341. unit_tests/sources/declarative/auth/__init__.py +0 -3
  342. unit_tests/sources/declarative/auth/test_oauth.py +0 -331
  343. unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
  344. unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
  345. unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
  346. unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
  347. unit_tests/sources/declarative/checks/__init__.py +0 -3
  348. unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
  349. unit_tests/sources/declarative/decoders/__init__.py +0 -0
  350. unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
  351. unit_tests/sources/declarative/external_component.py +0 -13
  352. unit_tests/sources/declarative/extractors/__init__.py +0 -3
  353. unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
  354. unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
  355. unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
  356. unit_tests/sources/declarative/incremental/__init__.py +0 -0
  357. unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
  358. unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
  359. unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
  360. unit_tests/sources/declarative/interpolation/__init__.py +0 -3
  361. unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
  362. unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
  363. unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
  364. unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
  365. unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
  366. unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
  367. unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
  368. unit_tests/sources/declarative/parsers/__init__.py +0 -3
  369. unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
  370. unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
  371. unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1841
  372. unit_tests/sources/declarative/parsers/testing_components.py +0 -36
  373. unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
  374. unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
  375. unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
  376. unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
  377. unit_tests/sources/declarative/requesters/__init__.py +0 -3
  378. unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
  379. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
  380. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
  381. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
  382. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
  383. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
  384. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
  385. unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
  386. unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
  387. unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
  388. unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
  389. unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
  390. unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
  391. unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
  392. unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
  393. unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
  394. unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
  395. unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
  396. unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
  397. unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
  398. unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
  399. unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
  400. unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
  401. unit_tests/sources/declarative/retrievers/__init__.py +0 -3
  402. unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
  403. unit_tests/sources/declarative/schema/__init__.py +0 -6
  404. unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
  405. unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
  406. unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
  407. unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
  408. unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
  409. unit_tests/sources/declarative/states/__init__.py +0 -3
  410. unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
  411. unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
  412. unit_tests/sources/declarative/test_create_partial.py +0 -83
  413. unit_tests/sources/declarative/test_declarative_stream.py +0 -103
  414. unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
  415. unit_tests/sources/declarative/test_types.py +0 -39
  416. unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
  417. unit_tests/sources/file_based/__init__.py +0 -0
  418. unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
  419. unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
  420. unit_tests/sources/file_based/config/__init__.py +0 -0
  421. unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
  422. unit_tests/sources/file_based/config/test_csv_format.py +0 -34
  423. unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
  424. unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
  425. unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
  426. unit_tests/sources/file_based/file_types/__init__.py +0 -0
  427. unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
  428. unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
  429. unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
  430. unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
  431. unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
  432. unit_tests/sources/file_based/helpers.py +0 -70
  433. unit_tests/sources/file_based/in_memory_files_source.py +0 -211
  434. unit_tests/sources/file_based/scenarios/__init__.py +0 -0
  435. unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
  436. unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
  437. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
  438. unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
  439. unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
  440. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
  441. unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
  442. unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
  443. unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
  444. unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
  445. unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
  446. unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
  447. unit_tests/sources/file_based/stream/__init__.py +0 -0
  448. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  449. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
  450. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
  451. unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
  452. unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
  453. unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
  454. unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
  455. unit_tests/sources/file_based/test_scenarios.py +0 -253
  456. unit_tests/sources/file_based/test_schema_helpers.py +0 -346
  457. unit_tests/sources/fixtures/__init__.py +0 -3
  458. unit_tests/sources/fixtures/source_test_fixture.py +0 -153
  459. unit_tests/sources/message/__init__.py +0 -0
  460. unit_tests/sources/message/test_repository.py +0 -153
  461. unit_tests/sources/streams/__init__.py +0 -0
  462. unit_tests/sources/streams/concurrent/__init__.py +0 -3
  463. unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
  464. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
  465. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
  466. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
  467. unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
  468. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
  469. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
  470. unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
  471. unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
  472. unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
  473. unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
  474. unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
  475. unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
  476. unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
  477. unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
  478. unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
  479. unit_tests/sources/streams/http/__init__.py +0 -0
  480. unit_tests/sources/streams/http/auth/__init__.py +0 -0
  481. unit_tests/sources/streams/http/auth/test_auth.py +0 -173
  482. unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
  483. unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
  484. unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
  485. unit_tests/sources/streams/http/test_http.py +0 -635
  486. unit_tests/sources/streams/test_availability_strategy.py +0 -70
  487. unit_tests/sources/streams/test_call_rate.py +0 -300
  488. unit_tests/sources/streams/test_stream_read.py +0 -405
  489. unit_tests/sources/streams/test_streams_core.py +0 -184
  490. unit_tests/sources/test_abstract_source.py +0 -1442
  491. unit_tests/sources/test_concurrent_source.py +0 -112
  492. unit_tests/sources/test_config.py +0 -92
  493. unit_tests/sources/test_connector_state_manager.py +0 -482
  494. unit_tests/sources/test_http_logger.py +0 -252
  495. unit_tests/sources/test_integration_source.py +0 -86
  496. unit_tests/sources/test_source.py +0 -684
  497. unit_tests/sources/test_source_read.py +0 -460
  498. unit_tests/test/__init__.py +0 -0
  499. unit_tests/test/mock_http/__init__.py +0 -0
  500. unit_tests/test/mock_http/test_matcher.py +0 -53
  501. unit_tests/test/mock_http/test_mocker.py +0 -214
  502. unit_tests/test/mock_http/test_request.py +0 -117
  503. unit_tests/test/mock_http/test_response_builder.py +0 -177
  504. unit_tests/test/test_entrypoint_wrapper.py +0 -240
  505. unit_tests/utils/__init__.py +0 -0
  506. unit_tests/utils/test_datetime_format_inferrer.py +0 -60
  507. unit_tests/utils/test_mapping_helpers.py +0 -54
  508. unit_tests/utils/test_message_utils.py +0 -91
  509. unit_tests/utils/test_rate_limiting.py +0 -26
  510. unit_tests/utils/test_schema_inferrer.py +0 -202
  511. unit_tests/utils/test_secret_utils.py +0 -135
  512. unit_tests/utils/test_stream_status_utils.py +0 -61
  513. unit_tests/utils/test_traced_exception.py +0 -107
  514. /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
  515. {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
  516. {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
  517. {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
  518. {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/LICENSE.txt +0 -0
@@ -1,1841 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- # mypy: ignore-errors
6
-
7
- import datetime
8
- from typing import Any, Mapping
9
-
10
- import pytest
11
- from airbyte_cdk.models import Level
12
- from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator
13
- from airbyte_cdk.sources.declarative.auth.token import (
14
- ApiKeyAuthenticator,
15
- BasicHttpAuthenticator,
16
- BearerAuthenticator,
17
- LegacySessionTokenAuthenticator,
18
- )
19
- from airbyte_cdk.sources.declarative.auth.token_provider import SessionTokenProvider
20
- from airbyte_cdk.sources.declarative.checks import CheckStream
21
- from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime
22
- from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
23
- from airbyte_cdk.sources.declarative.decoders import JsonDecoder
24
- from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordFilter, RecordSelector
25
- from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor, PerPartitionCursor
26
- from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
27
- from airbyte_cdk.sources.declarative.models import CheckStream as CheckStreamModel
28
- from airbyte_cdk.sources.declarative.models import CompositeErrorHandler as CompositeErrorHandlerModel
29
- from airbyte_cdk.sources.declarative.models import CustomErrorHandler as CustomErrorHandlerModel
30
- from airbyte_cdk.sources.declarative.models import CustomPartitionRouter as CustomPartitionRouterModel
31
- from airbyte_cdk.sources.declarative.models import CustomSchemaLoader as CustomSchemaLoaderModel
32
- from airbyte_cdk.sources.declarative.models import DatetimeBasedCursor as DatetimeBasedCursorModel
33
- from airbyte_cdk.sources.declarative.models import DeclarativeStream as DeclarativeStreamModel
34
- from airbyte_cdk.sources.declarative.models import DefaultPaginator as DefaultPaginatorModel
35
- from airbyte_cdk.sources.declarative.models import HttpRequester as HttpRequesterModel
36
- from airbyte_cdk.sources.declarative.models import ListPartitionRouter as ListPartitionRouterModel
37
- from airbyte_cdk.sources.declarative.models import OAuthAuthenticator as OAuthAuthenticatorModel
38
- from airbyte_cdk.sources.declarative.models import RecordSelector as RecordSelectorModel
39
- from airbyte_cdk.sources.declarative.models import SimpleRetriever as SimpleRetrieverModel
40
- from airbyte_cdk.sources.declarative.models import Spec as SpecModel
41
- from airbyte_cdk.sources.declarative.models import SubstreamPartitionRouter as SubstreamPartitionRouterModel
42
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import OffsetIncrement as OffsetIncrementModel
43
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import PageIncrement as PageIncrementModel
44
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import SelectiveAuthenticator
45
- from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer
46
- from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver
47
- from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory
48
- from airbyte_cdk.sources.declarative.partition_routers import ListPartitionRouter, SinglePartitionRouter, SubstreamPartitionRouter
49
- from airbyte_cdk.sources.declarative.requesters import HttpRequester
50
- from airbyte_cdk.sources.declarative.requesters.error_handlers import CompositeErrorHandler, DefaultErrorHandler, HttpResponseFilter
51
- from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import (
52
- ConstantBackoffStrategy,
53
- ExponentialBackoffStrategy,
54
- WaitTimeFromHeaderBackoffStrategy,
55
- WaitUntilTimeFromHeaderBackoffStrategy,
56
- )
57
- from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction
58
- from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator
59
- from airbyte_cdk.sources.declarative.requesters.paginators.strategies import (
60
- CursorPaginationStrategy,
61
- OffsetIncrement,
62
- PageIncrement,
63
- StopConditionPaginationStrategyDecorator,
64
- )
65
- from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType
66
- from airbyte_cdk.sources.declarative.requesters.request_options import InterpolatedRequestOptionsProvider
67
- from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
68
- from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
69
- from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever, SimpleRetrieverTestReadDecorator
70
- from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader
71
- from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
72
- from airbyte_cdk.sources.declarative.spec import Spec
73
- from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStreamSlicer
74
- from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields
75
- from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
76
- from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
77
- from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator
78
- from unit_tests.sources.declarative.parsers.testing_components import TestingCustomSubstreamPartitionRouter, TestingSomeComponent
79
-
80
- factory = ModelToComponentFactory()
81
-
82
- resolver = ManifestReferenceResolver()
83
-
84
- transformer = ManifestComponentTransformer()
85
-
86
- input_config = {"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}
87
-
88
-
89
- def test_create_check_stream():
90
- manifest = {"check": {"type": "CheckStream", "stream_names": ["list_stream"]}}
91
-
92
- check = factory.create_component(CheckStreamModel, manifest["check"], {})
93
-
94
- assert isinstance(check, CheckStream)
95
- assert check.stream_names == ["list_stream"]
96
-
97
-
98
- def test_create_component_type_mismatch():
99
- manifest = {"check": {"type": "MismatchType", "stream_names": ["list_stream"]}}
100
-
101
- with pytest.raises(ValueError):
102
- factory.create_component(CheckStreamModel, manifest["check"], {})
103
-
104
-
105
- def test_full_config_stream():
106
- content = """
107
- decoder:
108
- type: JsonDecoder
109
- extractor:
110
- type: DpathExtractor
111
- decoder: "#/decoder"
112
- selector:
113
- type: RecordSelector
114
- record_filter:
115
- type: RecordFilter
116
- condition: "{{ record['id'] > stream_state['id'] }}"
117
- metadata_paginator:
118
- type: DefaultPaginator
119
- page_size_option:
120
- type: RequestOption
121
- inject_into: request_parameter
122
- field_name: page_size
123
- page_token_option:
124
- type: RequestPath
125
- pagination_strategy:
126
- type: "CursorPagination"
127
- cursor_value: "{{ response._metadata.next }}"
128
- page_size: 10
129
- requester:
130
- type: HttpRequester
131
- url_base: "https://api.sendgrid.com/v3/"
132
- http_method: "GET"
133
- authenticator:
134
- type: BearerAuthenticator
135
- api_token: "{{ config['apikey'] }}"
136
- request_parameters:
137
- unit: "day"
138
- retriever:
139
- paginator:
140
- type: NoPagination
141
- partial_stream:
142
- type: DeclarativeStream
143
- schema_loader:
144
- type: JsonFileSchemaLoader
145
- file_path: "./source_sendgrid/schemas/{{ parameters.name }}.json"
146
- list_stream:
147
- $ref: "#/partial_stream"
148
- $parameters:
149
- name: "lists"
150
- extractor:
151
- $ref: "#/extractor"
152
- field_path: ["{{ parameters['name'] }}"]
153
- name: "lists"
154
- primary_key: "id"
155
- retriever:
156
- $ref: "#/retriever"
157
- requester:
158
- $ref: "#/requester"
159
- path: "{{ next_page_token['next_page_url'] }}"
160
- paginator:
161
- $ref: "#/metadata_paginator"
162
- record_selector:
163
- $ref: "#/selector"
164
- transformations:
165
- - type: AddFields
166
- fields:
167
- - path: ["extra"]
168
- value: "{{ response.to_add }}"
169
- incremental_sync:
170
- type: DatetimeBasedCursor
171
- start_datetime: "{{ config['start_time'] }}"
172
- end_datetime: "{{ config['end_time'] }}"
173
- step: "P10D"
174
- cursor_field: "created"
175
- cursor_granularity: "PT0.000001S"
176
- $parameters:
177
- datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
178
- check:
179
- type: CheckStream
180
- stream_names: ["list_stream"]
181
- spec:
182
- type: Spec
183
- documentation_url: https://airbyte.com/#yaml-from-manifest
184
- connection_specification:
185
- title: Test Spec
186
- type: object
187
- required:
188
- - api_key
189
- additionalProperties: false
190
- properties:
191
- api_key:
192
- type: string
193
- airbyte_secret: true
194
- title: API Key
195
- description: Test API Key
196
- order: 0
197
- advanced_auth:
198
- auth_flow_type: "oauth2.0"
199
- """
200
- parsed_manifest = YamlDeclarativeSource._parse(content)
201
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
202
- resolved_manifest["type"] = "DeclarativeSource"
203
- manifest = transformer.propagate_types_and_parameters("", resolved_manifest, {})
204
-
205
- stream_manifest = manifest["list_stream"]
206
- assert stream_manifest["type"] == "DeclarativeStream"
207
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
208
-
209
- assert isinstance(stream, DeclarativeStream)
210
- assert stream.primary_key == "id"
211
- assert stream.name == "lists"
212
- assert stream._stream_cursor_field.string == "created"
213
-
214
- assert isinstance(stream.schema_loader, JsonFileSchemaLoader)
215
- assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.json"
216
-
217
- assert len(stream.retriever.record_selector.transformations) == 1
218
- add_fields = stream.retriever.record_selector.transformations[0]
219
- assert isinstance(add_fields, AddFields)
220
- assert add_fields.fields[0].path == ["extra"]
221
- assert add_fields.fields[0].value.string == "{{ response.to_add }}"
222
-
223
- assert isinstance(stream.retriever, SimpleRetriever)
224
- assert stream.retriever.primary_key == stream.primary_key
225
- assert stream.retriever.name == stream.name
226
-
227
- assert isinstance(stream.retriever.record_selector, RecordSelector)
228
-
229
- assert isinstance(stream.retriever.record_selector.extractor, DpathExtractor)
230
- assert isinstance(stream.retriever.record_selector.extractor.decoder, JsonDecoder)
231
- assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_path] == ["lists"]
232
-
233
- assert isinstance(stream.retriever.record_selector.record_filter, RecordFilter)
234
- assert stream.retriever.record_selector.record_filter._filter_interpolator.condition == "{{ record['id'] > stream_state['id'] }}"
235
-
236
- assert isinstance(stream.retriever.paginator, DefaultPaginator)
237
- assert isinstance(stream.retriever.paginator.decoder, JsonDecoder)
238
- assert stream.retriever.paginator.page_size_option.field_name.eval(input_config) == "page_size"
239
- assert stream.retriever.paginator.page_size_option.inject_into == RequestOptionType.request_parameter
240
- assert isinstance(stream.retriever.paginator.page_token_option, RequestPath)
241
- assert stream.retriever.paginator.url_base.string == "https://api.sendgrid.com/v3/"
242
- assert stream.retriever.paginator.url_base.default == "https://api.sendgrid.com/v3/"
243
-
244
- assert isinstance(stream.retriever.paginator.pagination_strategy, CursorPaginationStrategy)
245
- assert isinstance(stream.retriever.paginator.pagination_strategy.decoder, JsonDecoder)
246
- assert stream.retriever.paginator.pagination_strategy.cursor_value.string == "{{ response._metadata.next }}"
247
- assert stream.retriever.paginator.pagination_strategy.cursor_value.default == "{{ response._metadata.next }}"
248
- assert stream.retriever.paginator.pagination_strategy.page_size == 10
249
-
250
- assert isinstance(stream.retriever.requester, HttpRequester)
251
- assert stream.retriever.requester.http_method == HttpMethod.GET
252
- assert stream.retriever.requester.name == stream.name
253
- assert stream.retriever.requester._path.string == "{{ next_page_token['next_page_url'] }}"
254
- assert stream.retriever.requester._path.default == "{{ next_page_token['next_page_url'] }}"
255
-
256
- assert isinstance(stream.retriever.requester.authenticator, BearerAuthenticator)
257
- assert stream.retriever.requester.authenticator.token_provider.get_token() == "verysecrettoken"
258
-
259
- assert isinstance(stream.retriever.requester.request_options_provider, InterpolatedRequestOptionsProvider)
260
- assert stream.retriever.requester.request_options_provider.request_parameters.get("unit") == "day"
261
-
262
- checker = factory.create_component(model_type=CheckStreamModel, component_definition=manifest["check"], config=input_config)
263
-
264
- assert isinstance(checker, CheckStream)
265
- streams_to_check = checker.stream_names
266
- assert len(streams_to_check) == 1
267
- assert list(streams_to_check)[0] == "list_stream"
268
-
269
- spec = factory.create_component(model_type=SpecModel, component_definition=manifest["spec"], config=input_config)
270
-
271
- assert isinstance(spec, Spec)
272
- documentation_url = spec.documentation_url
273
- connection_specification = spec.connection_specification
274
- assert documentation_url == "https://airbyte.com/#yaml-from-manifest"
275
- assert connection_specification["title"] == "Test Spec"
276
- assert connection_specification["required"] == ["api_key"]
277
- assert connection_specification["properties"]["api_key"] == {
278
- "type": "string",
279
- "airbyte_secret": True,
280
- "title": "API Key",
281
- "description": "Test API Key",
282
- "order": 0,
283
- }
284
- advanced_auth = spec.advanced_auth
285
- assert advanced_auth.auth_flow_type.value == "oauth2.0"
286
-
287
-
288
- def test_interpolate_config():
289
- content = """
290
- authenticator:
291
- type: OAuthAuthenticator
292
- client_id: "some_client_id"
293
- client_secret: "some_client_secret"
294
- token_refresh_endpoint: "https://api.sendgrid.com/v3/auth"
295
- refresh_token: "{{ config['apikey'] }}"
296
- refresh_request_body:
297
- body_field: "yoyoyo"
298
- interpolated_body_field: "{{ config['apikey'] }}"
299
- """
300
- parsed_manifest = YamlDeclarativeSource._parse(content)
301
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
302
- authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
303
-
304
- authenticator = factory.create_component(
305
- model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=input_config
306
- )
307
-
308
- assert isinstance(authenticator, DeclarativeOauth2Authenticator)
309
- assert authenticator._client_id.eval(input_config) == "some_client_id"
310
- assert authenticator._client_secret.string == "some_client_secret"
311
- assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth"
312
- assert authenticator._refresh_token.eval(input_config) == "verysecrettoken"
313
- assert authenticator._refresh_request_body.mapping == {"body_field": "yoyoyo", "interpolated_body_field": "{{ config['apikey'] }}"}
314
- assert authenticator.get_refresh_request_body() == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"}
315
-
316
-
317
- def test_interpolate_config_with_token_expiry_date_format():
318
- content = """
319
- authenticator:
320
- type: OAuthAuthenticator
321
- client_id: "some_client_id"
322
- client_secret: "some_client_secret"
323
- token_refresh_endpoint: "https://api.sendgrid.com/v3/auth"
324
- refresh_token: "{{ config['apikey'] }}"
325
- token_expiry_date_format: "%Y-%m-%d %H:%M:%S.%f+00:00"
326
- """
327
- parsed_manifest = YamlDeclarativeSource._parse(content)
328
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
329
- authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
330
-
331
- authenticator = factory.create_component(
332
- model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=input_config
333
- )
334
-
335
- assert isinstance(authenticator, DeclarativeOauth2Authenticator)
336
- assert authenticator.token_expiry_date_format == "%Y-%m-%d %H:%M:%S.%f+00:00"
337
- assert authenticator.token_expiry_is_time_of_expiration
338
- assert authenticator._client_id.eval(input_config) == "some_client_id"
339
- assert authenticator._client_secret.string == "some_client_secret"
340
- assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth"
341
-
342
-
343
- def test_single_use_oauth_branch():
344
- single_use_input_config = {
345
- "apikey": "verysecrettoken",
346
- "repos": ["airbyte", "airbyte-cloud"],
347
- "credentials": {"access_token": "access_token", "token_expiry_date": "1970-01-01"},
348
- }
349
-
350
- content = """
351
- authenticator:
352
- type: OAuthAuthenticator
353
- client_id: "some_client_id"
354
- client_secret: "some_client_secret"
355
- token_refresh_endpoint: "https://api.sendgrid.com/v3/auth"
356
- refresh_token: "{{ config['apikey'] }}"
357
- refresh_request_body:
358
- body_field: "yoyoyo"
359
- interpolated_body_field: "{{ config['apikey'] }}"
360
- refresh_token_updater:
361
- refresh_token_name: "the_refresh_token"
362
- refresh_token_config_path:
363
- - apikey
364
- """
365
- parsed_manifest = YamlDeclarativeSource._parse(content)
366
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
367
- authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
368
-
369
- authenticator: SingleUseRefreshTokenOauth2Authenticator = factory.create_component(
370
- model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=single_use_input_config
371
- )
372
-
373
- assert isinstance(authenticator, SingleUseRefreshTokenOauth2Authenticator)
374
- assert authenticator._client_id == "some_client_id"
375
- assert authenticator._client_secret == "some_client_secret"
376
- assert authenticator._token_refresh_endpoint == "https://api.sendgrid.com/v3/auth"
377
- assert authenticator._refresh_token == "verysecrettoken"
378
- assert authenticator._refresh_request_body == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"}
379
- assert authenticator._refresh_token_name == "the_refresh_token"
380
- assert authenticator._refresh_token_config_path == ["apikey"]
381
- # default values
382
- assert authenticator._access_token_config_path == ["credentials", "access_token"]
383
- assert authenticator._token_expiry_date_config_path == ["credentials", "token_expiry_date"]
384
-
385
-
386
- def test_list_based_stream_slicer_with_values_refd():
387
- content = """
388
- repositories: ["airbyte", "airbyte-cloud"]
389
- partition_router:
390
- type: ListPartitionRouter
391
- values: "#/repositories"
392
- cursor_field: repository
393
- """
394
- parsed_manifest = YamlDeclarativeSource._parse(content)
395
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
396
- partition_router_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["partition_router"], {})
397
-
398
- partition_router = factory.create_component(
399
- model_type=ListPartitionRouterModel, component_definition=partition_router_manifest, config=input_config
400
- )
401
-
402
- assert isinstance(partition_router, ListPartitionRouter)
403
- assert partition_router.values == ["airbyte", "airbyte-cloud"]
404
-
405
-
406
- def test_list_based_stream_slicer_with_values_defined_in_config():
407
- content = """
408
- partition_router:
409
- type: ListPartitionRouter
410
- values: "{{config['repos']}}"
411
- cursor_field: repository
412
- request_option:
413
- type: RequestOption
414
- inject_into: header
415
- field_name: repository
416
- """
417
- parsed_manifest = YamlDeclarativeSource._parse(content)
418
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
419
- partition_router_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["partition_router"], {})
420
-
421
- partition_router = factory.create_component(
422
- model_type=ListPartitionRouterModel, component_definition=partition_router_manifest, config=input_config
423
- )
424
-
425
- assert isinstance(partition_router, ListPartitionRouter)
426
- assert partition_router.values == ["airbyte", "airbyte-cloud"]
427
- assert partition_router.request_option.inject_into == RequestOptionType.header
428
- assert partition_router.request_option.field_name.eval(config=input_config) == "repository"
429
-
430
-
431
- def test_create_substream_partition_router():
432
- content = """
433
- schema_loader:
434
- file_path: "./source_sendgrid/schemas/{{ parameters['name'] }}.yaml"
435
- name: "{{ parameters['stream_name'] }}"
436
- retriever:
437
- requester:
438
- type: "HttpRequester"
439
- path: "kek"
440
- record_selector:
441
- extractor:
442
- field_path: []
443
- stream_A:
444
- type: DeclarativeStream
445
- name: "A"
446
- primary_key: "id"
447
- $parameters:
448
- retriever: "#/retriever"
449
- url_base: "https://airbyte.io"
450
- schema_loader: "#/schema_loader"
451
- stream_B:
452
- type: DeclarativeStream
453
- name: "B"
454
- primary_key: "id"
455
- $parameters:
456
- retriever: "#/retriever"
457
- url_base: "https://airbyte.io"
458
- schema_loader: "#/schema_loader"
459
- partition_router:
460
- type: SubstreamPartitionRouter
461
- parent_stream_configs:
462
- - stream: "#/stream_A"
463
- parent_key: id
464
- partition_field: repository_id
465
- request_option:
466
- type: RequestOption
467
- inject_into: request_parameter
468
- field_name: repository_id
469
- - stream: "#/stream_B"
470
- parent_key: someid
471
- partition_field: word_id
472
- """
473
- parsed_manifest = YamlDeclarativeSource._parse(content)
474
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
475
- partition_router_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["partition_router"], {})
476
-
477
- partition_router = factory.create_component(
478
- model_type=SubstreamPartitionRouterModel, component_definition=partition_router_manifest, config=input_config
479
- )
480
-
481
- assert isinstance(partition_router, SubstreamPartitionRouter)
482
- parent_stream_configs = partition_router.parent_stream_configs
483
- assert len(parent_stream_configs) == 2
484
- assert isinstance(parent_stream_configs[0].stream, DeclarativeStream)
485
- assert isinstance(parent_stream_configs[1].stream, DeclarativeStream)
486
-
487
- assert partition_router.parent_stream_configs[0].parent_key.eval({}) == "id"
488
- assert partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id"
489
- assert partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter
490
- assert partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id"
491
-
492
- assert partition_router.parent_stream_configs[1].parent_key.eval({}) == "someid"
493
- assert partition_router.parent_stream_configs[1].partition_field.eval({}) == "word_id"
494
- assert partition_router.parent_stream_configs[1].request_option is None
495
-
496
-
497
- def test_datetime_based_cursor():
498
- content = """
499
- incremental:
500
- type: DatetimeBasedCursor
501
- $parameters:
502
- datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
503
- start_datetime:
504
- type: MinMaxDatetime
505
- datetime: "{{ config['start_time'] }}"
506
- min_datetime: "{{ config['start_time'] + day_delta(2) }}"
507
- end_datetime: "{{ config['end_time'] }}"
508
- step: "P10D"
509
- cursor_field: "created"
510
- cursor_granularity: "PT0.000001S"
511
- lookback_window: "P5D"
512
- start_time_option:
513
- type: RequestOption
514
- inject_into: request_parameter
515
- field_name: "since_{{ config['cursor_field'] }}"
516
- end_time_option:
517
- type: RequestOption
518
- inject_into: body_json
519
- field_name: "before_{{ parameters['cursor_field'] }}"
520
- partition_field_start: star
521
- partition_field_end: en
522
- """
523
- parsed_manifest = YamlDeclarativeSource._parse(content)
524
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
525
- slicer_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["incremental"], {"cursor_field": "created_at"})
526
-
527
- stream_slicer = factory.create_component(model_type=DatetimeBasedCursorModel, component_definition=slicer_manifest, config=input_config)
528
-
529
- assert isinstance(stream_slicer, DatetimeBasedCursor)
530
- assert stream_slicer._step == datetime.timedelta(days=10)
531
- assert stream_slicer._cursor_field.string == "created"
532
- assert stream_slicer.cursor_granularity == "PT0.000001S"
533
- assert stream_slicer._lookback_window.string == "P5D"
534
- assert stream_slicer.start_time_option.inject_into == RequestOptionType.request_parameter
535
- assert stream_slicer.start_time_option.field_name.eval(config=input_config | {"cursor_field": "updated_at"}) == "since_updated_at"
536
- assert stream_slicer.end_time_option.inject_into == RequestOptionType.body_json
537
- assert stream_slicer.end_time_option.field_name.eval({}) == "before_created_at"
538
- assert stream_slicer._partition_field_start.eval({}) == "star"
539
- assert stream_slicer._partition_field_end.eval({}) == "en"
540
-
541
- assert isinstance(stream_slicer._start_datetime, MinMaxDatetime)
542
- assert stream_slicer.start_datetime._datetime_format == "%Y-%m-%dT%H:%M:%S.%f%z"
543
- assert stream_slicer.start_datetime.datetime.string == "{{ config['start_time'] }}"
544
- assert stream_slicer.start_datetime.min_datetime.string == "{{ config['start_time'] + day_delta(2) }}"
545
-
546
- assert isinstance(stream_slicer._end_datetime, MinMaxDatetime)
547
- assert stream_slicer._end_datetime.datetime.string == "{{ config['end_time'] }}"
548
-
549
-
550
- def test_stream_with_incremental_and_retriever_with_partition_router():
551
- content = """
552
- decoder:
553
- type: JsonDecoder
554
- extractor:
555
- type: DpathExtractor
556
- decoder: "#/decoder"
557
- selector:
558
- type: RecordSelector
559
- record_filter:
560
- type: RecordFilter
561
- condition: "{{ record['id'] > stream_state['id'] }}"
562
- requester:
563
- type: HttpRequester
564
- name: "{{ parameters['name'] }}"
565
- url_base: "https://api.sendgrid.com/v3/"
566
- http_method: "GET"
567
- authenticator:
568
- type: BearerAuthenticator
569
- api_token: "{{ config['apikey'] }}"
570
- request_parameters:
571
- unit: "day"
572
- list_stream:
573
- type: DeclarativeStream
574
- schema_loader:
575
- type: JsonFileSchemaLoader
576
- file_path: "./source_sendgrid/schemas/{{ parameters.name }}.json"
577
- incremental_sync:
578
- type: DatetimeBasedCursor
579
- $parameters:
580
- datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
581
- start_datetime: "{{ config['start_time'] }}"
582
- end_datetime: "{{ config['end_time'] }}"
583
- step: "P10D"
584
- cursor_field: "created"
585
- cursor_granularity: "PT0.000001S"
586
- lookback_window: "P5D"
587
- start_time_option:
588
- inject_into: request_parameter
589
- field_name: created[gte]
590
- end_time_option:
591
- inject_into: body_json
592
- field_name: end_time
593
- partition_field_start: star
594
- partition_field_end: en
595
- retriever:
596
- type: SimpleRetriever
597
- name: "{{ parameters['name'] }}"
598
- partition_router:
599
- type: ListPartitionRouter
600
- values: "{{config['repos']}}"
601
- cursor_field: a_key
602
- request_option:
603
- inject_into: header
604
- field_name: a_key
605
- paginator:
606
- type: DefaultPaginator
607
- page_size_option:
608
- inject_into: request_parameter
609
- field_name: page_size
610
- page_token_option:
611
- inject_into: path
612
- type: RequestPath
613
- pagination_strategy:
614
- type: "CursorPagination"
615
- cursor_value: "{{ response._metadata.next }}"
616
- page_size: 10
617
- requester:
618
- $ref: "#/requester"
619
- path: "{{ next_page_token['next_page_url'] }}"
620
- record_selector:
621
- $ref: "#/selector"
622
- $parameters:
623
- name: "lists"
624
- primary_key: "id"
625
- extractor:
626
- $ref: "#/extractor"
627
- field_path: ["{{ parameters['name'] }}"]
628
- """
629
-
630
- parsed_manifest = YamlDeclarativeSource._parse(content)
631
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
632
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["list_stream"], {})
633
-
634
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
635
-
636
- assert isinstance(stream, DeclarativeStream)
637
- assert isinstance(stream.retriever, SimpleRetriever)
638
- assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor)
639
-
640
- datetime_stream_slicer = stream.retriever.stream_slicer._cursor_factory.create()
641
- assert isinstance(datetime_stream_slicer, DatetimeBasedCursor)
642
- assert isinstance(datetime_stream_slicer._start_datetime, MinMaxDatetime)
643
- assert datetime_stream_slicer._start_datetime.datetime.string == "{{ config['start_time'] }}"
644
- assert isinstance(datetime_stream_slicer._end_datetime, MinMaxDatetime)
645
- assert datetime_stream_slicer._end_datetime.datetime.string == "{{ config['end_time'] }}"
646
- assert datetime_stream_slicer.step == "P10D"
647
- assert datetime_stream_slicer._cursor_field.string == "created"
648
-
649
- list_stream_slicer = stream.retriever.stream_slicer._partition_router
650
- assert isinstance(list_stream_slicer, ListPartitionRouter)
651
- assert list_stream_slicer.values == ["airbyte", "airbyte-cloud"]
652
- assert list_stream_slicer._cursor_field.string == "a_key"
653
-
654
-
655
- def test_incremental_data_feed():
656
- content = """
657
- selector:
658
- type: RecordSelector
659
- extractor:
660
- type: DpathExtractor
661
- field_path: ["extractor_path"]
662
- record_filter:
663
- type: RecordFilter
664
- condition: "{{ record['id'] > stream_state['id'] }}"
665
- requester:
666
- type: HttpRequester
667
- name: "{{ parameters['name'] }}"
668
- url_base: "https://api.sendgrid.com/v3/"
669
- http_method: "GET"
670
- list_stream:
671
- type: DeclarativeStream
672
- incremental_sync:
673
- type: DatetimeBasedCursor
674
- $parameters:
675
- datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
676
- start_datetime: "{{ config['start_time'] }}"
677
- cursor_field: "created"
678
- is_data_feed: true
679
- retriever:
680
- type: SimpleRetriever
681
- name: "{{ parameters['name'] }}"
682
- paginator:
683
- type: DefaultPaginator
684
- pagination_strategy:
685
- type: "CursorPagination"
686
- cursor_value: "{{ response._metadata.next }}"
687
- page_size: 10
688
- requester:
689
- $ref: "#/requester"
690
- path: "/"
691
- record_selector:
692
- $ref: "#/selector"
693
- $parameters:
694
- name: "lists"
695
- """
696
-
697
- parsed_manifest = YamlDeclarativeSource._parse(content)
698
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
699
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["list_stream"], {})
700
-
701
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
702
-
703
- assert isinstance(stream.retriever.paginator.pagination_strategy, StopConditionPaginationStrategyDecorator)
704
-
705
-
706
- def test_given_data_feed_and_incremental_then_raise_error():
707
- content = """
708
- incremental_sync:
709
- type: DatetimeBasedCursor
710
- $parameters:
711
- datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
712
- start_datetime: "{{ config['start_time'] }}"
713
- end_datetime: "2023-01-01"
714
- cursor_field: "created"
715
- is_data_feed: true"""
716
-
717
- parsed_incremental_sync = YamlDeclarativeSource._parse(content)
718
- resolved_incremental_sync = resolver.preprocess_manifest(parsed_incremental_sync)
719
- datetime_based_cursor_definition = transformer.propagate_types_and_parameters("", resolved_incremental_sync["incremental_sync"], {})
720
-
721
- with pytest.raises(ValueError):
722
- factory.create_component(
723
- model_type=DatetimeBasedCursorModel, component_definition=datetime_based_cursor_definition, config=input_config
724
- )
725
-
726
-
727
- @pytest.mark.parametrize(
728
- "test_name, record_selector, expected_runtime_selector",
729
- [("test_static_record_selector", "result", "result"), ("test_options_record_selector", "{{ parameters['name'] }}", "lists")],
730
- )
731
- def test_create_record_selector(test_name, record_selector, expected_runtime_selector):
732
- content = f"""
733
- extractor:
734
- type: DpathExtractor
735
- selector:
736
- $parameters:
737
- name: "lists"
738
- type: RecordSelector
739
- record_filter:
740
- type: RecordFilter
741
- condition: "{{{{ record['id'] > stream_state['id'] }}}}"
742
- extractor:
743
- $ref: "#/extractor"
744
- field_path: ["{record_selector}"]
745
- """
746
- parsed_manifest = YamlDeclarativeSource._parse(content)
747
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
748
- selector_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["selector"], {})
749
-
750
- selector = factory.create_component(
751
- model_type=RecordSelectorModel, component_definition=selector_manifest, transformations=[], config=input_config
752
- )
753
-
754
- assert isinstance(selector, RecordSelector)
755
- assert isinstance(selector.extractor, DpathExtractor)
756
- assert [fp.eval(input_config) for fp in selector.extractor.field_path] == [expected_runtime_selector]
757
- assert isinstance(selector.record_filter, RecordFilter)
758
- assert selector.record_filter.condition == "{{ record['id'] > stream_state['id'] }}"
759
-
760
-
761
- @pytest.mark.parametrize(
762
- "test_name, error_handler, expected_backoff_strategy_type",
763
- [
764
- (
765
- "test_create_requester_constant_error_handler",
766
- """
767
- error_handler:
768
- backoff_strategies:
769
- - type: "ConstantBackoffStrategy"
770
- backoff_time_in_seconds: 5
771
- """,
772
- ConstantBackoffStrategy,
773
- ),
774
- (
775
- "test_create_requester_exponential_error_handler",
776
- """
777
- error_handler:
778
- backoff_strategies:
779
- - type: "ExponentialBackoffStrategy"
780
- factor: 5
781
- """,
782
- ExponentialBackoffStrategy,
783
- ),
784
- (
785
- "test_create_requester_wait_time_from_header_error_handler",
786
- """
787
- error_handler:
788
- backoff_strategies:
789
- - type: "WaitTimeFromHeader"
790
- header: "a_header"
791
- """,
792
- WaitTimeFromHeaderBackoffStrategy,
793
- ),
794
- (
795
- "test_create_requester_wait_time_until_from_header_error_handler",
796
- """
797
- error_handler:
798
- backoff_strategies:
799
- - type: "WaitUntilTimeFromHeader"
800
- header: "a_header"
801
- """,
802
- WaitUntilTimeFromHeaderBackoffStrategy,
803
- ),
804
- ("test_create_requester_no_error_handler", """""", ExponentialBackoffStrategy),
805
- ],
806
- )
807
- def test_create_requester(test_name, error_handler, expected_backoff_strategy_type):
808
- content = f"""
809
- requester:
810
- type: HttpRequester
811
- path: "/v3/marketing/lists"
812
- $parameters:
813
- name: 'lists'
814
- url_base: "https://api.sendgrid.com"
815
- authenticator:
816
- type: "BasicHttpAuthenticator"
817
- username: "{{{{ parameters.name}}}}"
818
- password: "{{{{ config.apikey }}}}"
819
- request_parameters:
820
- a_parameter: "something_here"
821
- request_headers:
822
- header: header_value
823
- {error_handler}
824
- """
825
- name = "name"
826
- parsed_manifest = YamlDeclarativeSource._parse(content)
827
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
828
- requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {})
829
-
830
- selector = factory.create_component(
831
- model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name
832
- )
833
-
834
- assert isinstance(selector, HttpRequester)
835
- assert selector.http_method == HttpMethod.GET
836
- assert selector.name == "name"
837
- assert selector._path.string == "/v3/marketing/lists"
838
- assert selector._url_base.string == "https://api.sendgrid.com"
839
-
840
- assert isinstance(selector.error_handler, DefaultErrorHandler)
841
- assert len(selector.error_handler.backoff_strategies) == 1
842
- assert isinstance(selector.error_handler.backoff_strategies[0], expected_backoff_strategy_type)
843
-
844
- assert isinstance(selector.authenticator, BasicHttpAuthenticator)
845
- assert selector.authenticator._username.eval(input_config) == "lists"
846
- assert selector.authenticator._password.eval(input_config) == "verysecrettoken"
847
-
848
- assert isinstance(selector._request_options_provider, InterpolatedRequestOptionsProvider)
849
- assert selector._request_options_provider._parameter_interpolator._interpolator.mapping["a_parameter"] == "something_here"
850
- assert selector._request_options_provider._headers_interpolator._interpolator.mapping["header"] == "header_value"
851
-
852
-
853
- def test_create_request_with_leacy_session_authenticator():
854
- content = """
855
- requester:
856
- type: HttpRequester
857
- path: "/v3/marketing/lists"
858
- $parameters:
859
- name: 'lists'
860
- url_base: "https://api.sendgrid.com"
861
- authenticator:
862
- type: "LegacySessionTokenAuthenticator"
863
- username: "{{ parameters.name}}"
864
- password: "{{ config.apikey }}"
865
- login_url: "login"
866
- header: "token"
867
- session_token_response_key: "session"
868
- validate_session_url: validate
869
- request_parameters:
870
- a_parameter: "something_here"
871
- request_headers:
872
- header: header_value
873
- """
874
- name = "name"
875
- parsed_manifest = YamlDeclarativeSource._parse(content)
876
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
877
- requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {})
878
-
879
- selector = factory.create_component(
880
- model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name
881
- )
882
-
883
- assert isinstance(selector, HttpRequester)
884
- assert isinstance(selector.authenticator, LegacySessionTokenAuthenticator)
885
- assert selector.authenticator._username.eval(input_config) == "lists"
886
- assert selector.authenticator._password.eval(input_config) == "verysecrettoken"
887
- assert selector.authenticator._api_url.eval(input_config) == "https://api.sendgrid.com"
888
-
889
-
890
- def test_create_request_with_session_authenticator():
891
- content = """
892
- requester:
893
- type: HttpRequester
894
- path: "/v3/marketing/lists"
895
- $parameters:
896
- name: 'lists'
897
- url_base: "https://api.sendgrid.com"
898
- authenticator:
899
- type: SessionTokenAuthenticator
900
- expiration_duration: P10D
901
- login_requester:
902
- path: /session
903
- type: HttpRequester
904
- url_base: 'https://api.sendgrid.com'
905
- http_method: POST
906
- request_body_json:
907
- password: '{{ config.apikey }}'
908
- username: '{{ parameters.name }}'
909
- session_token_path:
910
- - id
911
- request_authentication:
912
- type: ApiKey
913
- inject_into:
914
- type: RequestOption
915
- field_name: X-Metabase-Session
916
- inject_into: header
917
- request_parameters:
918
- a_parameter: "something_here"
919
- request_headers:
920
- header: header_value
921
- """
922
- name = "name"
923
- parsed_manifest = YamlDeclarativeSource._parse(content)
924
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
925
- requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {})
926
-
927
- selector = factory.create_component(
928
- model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name
929
- )
930
-
931
- assert isinstance(selector.authenticator, ApiKeyAuthenticator)
932
- assert isinstance(selector.authenticator.token_provider, SessionTokenProvider)
933
- assert selector.authenticator.token_provider.session_token_path == ["id"]
934
- assert isinstance(selector.authenticator.token_provider.login_requester, HttpRequester)
935
- assert selector.authenticator.token_provider.session_token_path == ["id"]
936
- assert selector.authenticator.token_provider.login_requester._url_base.eval(input_config) == "https://api.sendgrid.com"
937
- assert selector.authenticator.token_provider.login_requester.get_request_body_json() == {
938
- "username": "lists",
939
- "password": "verysecrettoken",
940
- }
941
-
942
-
943
- @pytest.mark.parametrize(
944
- "input_config, expected_authenticator_class",
945
- [
946
- pytest.param(
947
- {"auth": {"type": "token"}, "credentials": {"api_key": "some_key"}},
948
- ApiKeyAuthenticator,
949
- id="test_create_requester_with_selective_authenticator_and_token_selected",
950
- ),
951
- pytest.param(
952
- {"auth": {"type": "oauth"}, "credentials": {"client_id": "ABC"}},
953
- DeclarativeOauth2Authenticator,
954
- id="test_create_requester_with_selective_authenticator_and_oauth_selected",
955
- ),
956
- ],
957
- )
958
- def test_create_requester_with_selective_authenticator(input_config, expected_authenticator_class):
959
- content = """
960
- authenticator:
961
- type: SelectiveAuthenticator
962
- authenticator_selection_path:
963
- - auth
964
- - type
965
- authenticators:
966
- token:
967
- type: ApiKeyAuthenticator
968
- header: "Authorization"
969
- api_token: "api_key={{ config['credentials']['api_key'] }}"
970
- oauth:
971
- type: OAuthAuthenticator
972
- token_refresh_endpoint: https://api.url.com
973
- client_id: "{{ config['credentials']['client_id'] }}"
974
- client_secret: some_secret
975
- refresh_token: some_token
976
- """
977
- name = "name"
978
- parsed_manifest = YamlDeclarativeSource._parse(content)
979
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
980
- authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
981
-
982
- authenticator = factory.create_component(
983
- model_type=SelectiveAuthenticator, component_definition=authenticator_manifest, config=input_config, name=name
984
- )
985
-
986
- assert isinstance(authenticator, expected_authenticator_class)
987
-
988
-
989
- def test_create_composite_error_handler():
990
- content = """
991
- error_handler:
992
- type: "CompositeErrorHandler"
993
- error_handlers:
994
- - response_filters:
995
- - predicate: "{{ 'code' in response }}"
996
- action: RETRY
997
- - response_filters:
998
- - http_codes: [ 403 ]
999
- action: RETRY
1000
- """
1001
- parsed_manifest = YamlDeclarativeSource._parse(content)
1002
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1003
- error_handler_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["error_handler"], {})
1004
-
1005
- error_handler = factory.create_component(
1006
- model_type=CompositeErrorHandlerModel, component_definition=error_handler_manifest, config=input_config
1007
- )
1008
-
1009
- assert isinstance(error_handler, CompositeErrorHandler)
1010
- assert len(error_handler.error_handlers) == 2
1011
-
1012
- error_handler_0 = error_handler.error_handlers[0]
1013
- assert isinstance(error_handler_0, DefaultErrorHandler)
1014
- assert isinstance(error_handler_0.response_filters[0], HttpResponseFilter)
1015
- assert error_handler_0.response_filters[0].predicate.condition == "{{ 'code' in response }}"
1016
- assert error_handler_0.response_filters[0].action == ResponseAction.RETRY
1017
-
1018
- error_handler_1 = error_handler.error_handlers[1]
1019
- assert isinstance(error_handler_1, DefaultErrorHandler)
1020
- assert isinstance(error_handler_1.response_filters[0], HttpResponseFilter)
1021
- assert error_handler_1.response_filters[0].http_codes == {403}
1022
- assert error_handler_1.response_filters[0].action == ResponseAction.RETRY
1023
-
1024
-
1025
- # This might be a better test for the manifest transformer but also worth testing end-to-end here as well
1026
- def test_config_with_defaults():
1027
- content = """
1028
- lists_stream:
1029
- type: "DeclarativeStream"
1030
- name: "lists"
1031
- primary_key: id
1032
- $parameters:
1033
- name: "lists"
1034
- url_base: "https://api.sendgrid.com"
1035
- schema_loader:
1036
- name: "{{ parameters.stream_name }}"
1037
- file_path: "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
1038
- retriever:
1039
- paginator:
1040
- type: "DefaultPaginator"
1041
- page_size_option:
1042
- type: RequestOption
1043
- inject_into: request_parameter
1044
- field_name: page_size
1045
- page_token_option:
1046
- type: RequestPath
1047
- pagination_strategy:
1048
- type: "CursorPagination"
1049
- cursor_value: "{{ response._metadata.next }}"
1050
- page_size: 10
1051
- requester:
1052
- path: "/v3/marketing/lists"
1053
- authenticator:
1054
- type: "BearerAuthenticator"
1055
- api_token: "{{ config.apikey }}"
1056
- request_parameters:
1057
- page_size: 10
1058
- record_selector:
1059
- extractor:
1060
- field_path: ["result"]
1061
- streams:
1062
- - "#/lists_stream"
1063
- """
1064
- parsed_manifest = YamlDeclarativeSource._parse(content)
1065
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1066
- resolved_manifest["type"] = "DeclarativeSource"
1067
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["lists_stream"], {})
1068
-
1069
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
1070
-
1071
- assert isinstance(stream, DeclarativeStream)
1072
- assert stream.primary_key == "id"
1073
- assert stream.name == "lists"
1074
- assert isinstance(stream.retriever, SimpleRetriever)
1075
- assert stream.retriever.name == stream.name
1076
- assert stream.retriever.primary_key == stream.primary_key
1077
-
1078
- assert isinstance(stream.schema_loader, JsonFileSchemaLoader)
1079
- assert stream.schema_loader.file_path.string == "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
1080
- assert stream.schema_loader.file_path.default == "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
1081
-
1082
- assert isinstance(stream.retriever.requester, HttpRequester)
1083
- assert stream.retriever.requester.http_method == HttpMethod.GET
1084
-
1085
- assert isinstance(stream.retriever.requester.authenticator, BearerAuthenticator)
1086
- assert stream.retriever.requester.authenticator.token_provider.get_token() == "verysecrettoken"
1087
-
1088
- assert isinstance(stream.retriever.record_selector, RecordSelector)
1089
- assert isinstance(stream.retriever.record_selector.extractor, DpathExtractor)
1090
- assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_path] == ["result"]
1091
-
1092
- assert isinstance(stream.retriever.paginator, DefaultPaginator)
1093
- assert stream.retriever.paginator.url_base.string == "https://api.sendgrid.com"
1094
- assert stream.retriever.paginator.pagination_strategy.get_page_size() == 10
1095
-
1096
-
1097
- def test_create_default_paginator():
1098
- content = """
1099
- paginator:
1100
- type: "DefaultPaginator"
1101
- page_size_option:
1102
- type: RequestOption
1103
- inject_into: request_parameter
1104
- field_name: page_size
1105
- page_token_option:
1106
- type: RequestPath
1107
- pagination_strategy:
1108
- type: "CursorPagination"
1109
- page_size: 50
1110
- cursor_value: "{{ response._metadata.next }}"
1111
- """
1112
- parsed_manifest = YamlDeclarativeSource._parse(content)
1113
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1114
- paginator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["paginator"], {})
1115
-
1116
- paginator = factory.create_component(
1117
- model_type=DefaultPaginatorModel, component_definition=paginator_manifest, config=input_config, url_base="https://airbyte.io"
1118
- )
1119
-
1120
- assert isinstance(paginator, DefaultPaginator)
1121
- assert paginator.url_base.string == "https://airbyte.io"
1122
-
1123
- assert isinstance(paginator.pagination_strategy, CursorPaginationStrategy)
1124
- assert paginator.pagination_strategy.page_size == 50
1125
- assert paginator.pagination_strategy.cursor_value.string == "{{ response._metadata.next }}"
1126
-
1127
- assert isinstance(paginator.page_size_option, RequestOption)
1128
- assert paginator.page_size_option.inject_into == RequestOptionType.request_parameter
1129
- assert paginator.page_size_option.field_name.eval(config=input_config) == "page_size"
1130
-
1131
- assert isinstance(paginator.page_token_option, RequestPath)
1132
-
1133
-
1134
- @pytest.mark.parametrize(
1135
- "manifest, field_name, expected_value, expected_error",
1136
- [
1137
- pytest.param(
1138
- {
1139
- "type": "CustomErrorHandler",
1140
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1141
- "subcomponent_field_with_hint": {"type": "DpathExtractor", "field_path": []},
1142
- },
1143
- "subcomponent_field_with_hint",
1144
- DpathExtractor(field_path=[], config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, parameters={}),
1145
- None,
1146
- id="test_create_custom_component_with_subcomponent_that_must_be_parsed",
1147
- ),
1148
- pytest.param(
1149
- {
1150
- "type": "CustomErrorHandler",
1151
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1152
- "subcomponent_field_with_hint": {"field_path": []},
1153
- },
1154
- "subcomponent_field_with_hint",
1155
- DpathExtractor(field_path=[], config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, parameters={}),
1156
- None,
1157
- id="test_create_custom_component_with_subcomponent_that_must_infer_type_from_explicit_hints",
1158
- ),
1159
- pytest.param(
1160
- {
1161
- "type": "CustomErrorHandler",
1162
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1163
- "basic_field": "expected",
1164
- },
1165
- "basic_field",
1166
- "expected",
1167
- None,
1168
- id="test_create_custom_component_with_built_in_type",
1169
- ),
1170
- pytest.param(
1171
- {
1172
- "type": "CustomErrorHandler",
1173
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1174
- "optional_subcomponent_field": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "destination"},
1175
- },
1176
- "optional_subcomponent_field",
1177
- RequestOption(inject_into=RequestOptionType.request_parameter, field_name="destination", parameters={}),
1178
- None,
1179
- id="test_create_custom_component_with_subcomponent_wrapped_in_optional",
1180
- ),
1181
- pytest.param(
1182
- {
1183
- "type": "CustomErrorHandler",
1184
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1185
- "list_of_subcomponents": [
1186
- {"inject_into": "header", "field_name": "store_me"},
1187
- {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "destination"},
1188
- ],
1189
- },
1190
- "list_of_subcomponents",
1191
- [
1192
- RequestOption(inject_into=RequestOptionType.header, field_name="store_me", parameters={}),
1193
- RequestOption(inject_into=RequestOptionType.request_parameter, field_name="destination", parameters={}),
1194
- ],
1195
- None,
1196
- id="test_create_custom_component_with_subcomponent_wrapped_in_list",
1197
- ),
1198
- pytest.param(
1199
- {
1200
- "type": "CustomErrorHandler",
1201
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1202
- "without_hint": {"inject_into": "request_parameter", "field_name": "missing_hint"},
1203
- },
1204
- "without_hint",
1205
- None,
1206
- None,
1207
- id="test_create_custom_component_with_subcomponent_without_type_hints",
1208
- ),
1209
- pytest.param(
1210
- {
1211
- "type": "CustomErrorHandler",
1212
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1213
- "paginator": {
1214
- "type": "DefaultPaginator",
1215
- "pagination_strategy": {"type": "OffsetIncrement", "page_size": 10},
1216
- "$parameters": {"url_base": "https://physical_100.com"},
1217
- },
1218
- },
1219
- "paginator",
1220
- DefaultPaginator(
1221
- pagination_strategy=OffsetIncrement(
1222
- page_size=10, config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, parameters={}
1223
- ),
1224
- url_base="https://physical_100.com",
1225
- config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]},
1226
- parameters={},
1227
- ),
1228
- None,
1229
- id="test_create_custom_component_with_subcomponent_that_uses_parameters",
1230
- ),
1231
- pytest.param(
1232
- {
1233
- "type": "CustomErrorHandler",
1234
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1235
- "paginator": {
1236
- "type": "DefaultPaginator",
1237
- "pagination_strategy": {"type": "OffsetIncrement", "page_size": 10},
1238
- },
1239
- },
1240
- "paginator",
1241
- None,
1242
- ValueError,
1243
- id="test_create_custom_component_missing_required_field_emits_error",
1244
- ),
1245
- pytest.param(
1246
- {
1247
- "type": "CustomErrorHandler",
1248
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.NonExistingClass",
1249
- "paginator": {
1250
- "type": "DefaultPaginator",
1251
- "pagination_strategy": {"type": "OffsetIncrement", "page_size": 10},
1252
- },
1253
- },
1254
- "paginator",
1255
- None,
1256
- ValueError,
1257
- id="test_create_custom_component_non_existing_class_raises_value_error",
1258
- ),
1259
- ],
1260
- )
1261
- def test_create_custom_components(manifest, field_name, expected_value, expected_error):
1262
- if expected_error:
1263
- with pytest.raises(expected_error):
1264
- factory.create_component(CustomErrorHandlerModel, manifest, input_config)
1265
- else:
1266
- custom_component = factory.create_component(CustomErrorHandlerModel, manifest, input_config)
1267
- assert isinstance(custom_component, TestingSomeComponent)
1268
-
1269
- assert isinstance(getattr(custom_component, field_name), type(expected_value))
1270
- assert getattr(custom_component, field_name) == expected_value
1271
-
1272
-
1273
- def test_custom_components_do_not_contain_extra_fields():
1274
- custom_substream_partition_router_manifest = {
1275
- "type": "CustomPartitionRouter",
1276
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingCustomSubstreamPartitionRouter",
1277
- "custom_field": "here",
1278
- "extra_field_to_exclude": "should_not_pass_as_parameter",
1279
- "custom_pagination_strategy": {"type": "PageIncrement", "page_size": 100},
1280
- "parent_stream_configs": [
1281
- {
1282
- "type": "ParentStreamConfig",
1283
- "stream": {
1284
- "type": "DeclarativeStream",
1285
- "name": "a_parent",
1286
- "primary_key": "id",
1287
- "retriever": {
1288
- "type": "SimpleRetriever",
1289
- "record_selector": {
1290
- "type": "RecordSelector",
1291
- "extractor": {"type": "DpathExtractor", "field_path": []},
1292
- },
1293
- "requester": {"type": "HttpRequester", "url_base": "https://airbyte.io", "path": "some"},
1294
- },
1295
- "schema_loader": {
1296
- "type": "JsonFileSchemaLoader",
1297
- "file_path": "./source_sendgrid/schemas/{{ parameters['name'] }}.yaml",
1298
- },
1299
- },
1300
- "parent_key": "id",
1301
- "partition_field": "repository_id",
1302
- "request_option": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "repository_id"},
1303
- }
1304
- ],
1305
- }
1306
-
1307
- custom_substream_partition_router = factory.create_component(
1308
- CustomPartitionRouterModel, custom_substream_partition_router_manifest, input_config
1309
- )
1310
- assert isinstance(custom_substream_partition_router, TestingCustomSubstreamPartitionRouter)
1311
-
1312
- assert len(custom_substream_partition_router.parent_stream_configs) == 1
1313
- assert custom_substream_partition_router.parent_stream_configs[0].parent_key.eval({}) == "id"
1314
- assert custom_substream_partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id"
1315
- assert custom_substream_partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter
1316
- assert custom_substream_partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id"
1317
-
1318
- assert isinstance(custom_substream_partition_router.custom_pagination_strategy, PageIncrement)
1319
- assert custom_substream_partition_router.custom_pagination_strategy.page_size == 100
1320
-
1321
-
1322
- def test_parse_custom_component_fields_if_subcomponent():
1323
- custom_substream_partition_router_manifest = {
1324
- "type": "CustomPartitionRouter",
1325
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingCustomSubstreamPartitionRouter",
1326
- "custom_field": "here",
1327
- "custom_pagination_strategy": {"type": "PageIncrement", "page_size": 100},
1328
- "parent_stream_configs": [
1329
- {
1330
- "type": "ParentStreamConfig",
1331
- "stream": {
1332
- "type": "DeclarativeStream",
1333
- "name": "a_parent",
1334
- "primary_key": "id",
1335
- "retriever": {
1336
- "type": "SimpleRetriever",
1337
- "record_selector": {
1338
- "type": "RecordSelector",
1339
- "extractor": {"type": "DpathExtractor", "field_path": []},
1340
- },
1341
- "requester": {"type": "HttpRequester", "url_base": "https://airbyte.io", "path": "some"},
1342
- },
1343
- "schema_loader": {
1344
- "type": "JsonFileSchemaLoader",
1345
- "file_path": "./source_sendgrid/schemas/{{ parameters['name'] }}.yaml",
1346
- },
1347
- },
1348
- "parent_key": "id",
1349
- "partition_field": "repository_id",
1350
- "request_option": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "repository_id"},
1351
- }
1352
- ],
1353
- }
1354
-
1355
- custom_substream_partition_router = factory.create_component(
1356
- CustomPartitionRouterModel, custom_substream_partition_router_manifest, input_config
1357
- )
1358
- assert isinstance(custom_substream_partition_router, TestingCustomSubstreamPartitionRouter)
1359
- assert custom_substream_partition_router.custom_field == "here"
1360
-
1361
- assert len(custom_substream_partition_router.parent_stream_configs) == 1
1362
- assert custom_substream_partition_router.parent_stream_configs[0].parent_key.eval({}) == "id"
1363
- assert custom_substream_partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id"
1364
- assert custom_substream_partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter
1365
- assert custom_substream_partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id"
1366
-
1367
- assert isinstance(custom_substream_partition_router.custom_pagination_strategy, PageIncrement)
1368
- assert custom_substream_partition_router.custom_pagination_strategy.page_size == 100
1369
-
1370
-
1371
- class TestCreateTransformations:
1372
- # the tabbing matters
1373
- base_parameters = """
1374
- name: "lists"
1375
- primary_key: id
1376
- url_base: "https://api.sendgrid.com"
1377
- schema_loader:
1378
- name: "{{ parameters.name }}"
1379
- file_path: "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
1380
- retriever:
1381
- requester:
1382
- name: "{{ parameters.name }}"
1383
- path: "/v3/marketing/lists"
1384
- request_parameters:
1385
- page_size: 10
1386
- record_selector:
1387
- extractor:
1388
- field_path: ["result"]
1389
- """
1390
-
1391
- def test_no_transformations(self):
1392
- content = f"""
1393
- the_stream:
1394
- type: DeclarativeStream
1395
- $parameters:
1396
- {self.base_parameters}
1397
- """
1398
- parsed_manifest = YamlDeclarativeSource._parse(content)
1399
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1400
- resolved_manifest["type"] = "DeclarativeSource"
1401
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["the_stream"], {})
1402
-
1403
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
1404
-
1405
- assert isinstance(stream, DeclarativeStream)
1406
- assert [] == stream.retriever.record_selector.transformations
1407
-
1408
- def test_remove_fields(self):
1409
- content = f"""
1410
- the_stream:
1411
- type: DeclarativeStream
1412
- $parameters:
1413
- {self.base_parameters}
1414
- transformations:
1415
- - type: RemoveFields
1416
- field_pointers:
1417
- - ["path", "to", "field1"]
1418
- - ["path2"]
1419
- """
1420
- parsed_manifest = YamlDeclarativeSource._parse(content)
1421
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1422
- resolved_manifest["type"] = "DeclarativeSource"
1423
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["the_stream"], {})
1424
-
1425
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
1426
-
1427
- assert isinstance(stream, DeclarativeStream)
1428
- expected = [RemoveFields(field_pointers=[["path", "to", "field1"], ["path2"]], parameters={})]
1429
- assert stream.retriever.record_selector.transformations == expected
1430
-
1431
- def test_add_fields_no_value_type(self):
1432
- content = f"""
1433
- the_stream:
1434
- type: DeclarativeStream
1435
- $parameters:
1436
- {self.base_parameters}
1437
- transformations:
1438
- - type: AddFields
1439
- fields:
1440
- - path: ["field1"]
1441
- value: "static_value"
1442
- """
1443
- expected = [
1444
- AddFields(
1445
- fields=[
1446
- AddedFieldDefinition(
1447
- path=["field1"],
1448
- value=InterpolatedString(string="static_value", default="static_value", parameters={}),
1449
- value_type=None,
1450
- parameters={},
1451
- )
1452
- ],
1453
- parameters={},
1454
- )
1455
- ]
1456
- self._test_add_fields(content, expected)
1457
-
1458
- def test_add_fields_value_type_is_string(self):
1459
- content = f"""
1460
- the_stream:
1461
- type: DeclarativeStream
1462
- $parameters:
1463
- {self.base_parameters}
1464
- transformations:
1465
- - type: AddFields
1466
- fields:
1467
- - path: ["field1"]
1468
- value: "static_value"
1469
- value_type: string
1470
- """
1471
- expected = [
1472
- AddFields(
1473
- fields=[
1474
- AddedFieldDefinition(
1475
- path=["field1"],
1476
- value=InterpolatedString(string="static_value", default="static_value", parameters={}),
1477
- value_type=str,
1478
- parameters={},
1479
- )
1480
- ],
1481
- parameters={},
1482
- )
1483
- ]
1484
- self._test_add_fields(content, expected)
1485
-
1486
- def test_add_fields_value_type_is_number(self):
1487
- content = f"""
1488
- the_stream:
1489
- type: DeclarativeStream
1490
- $parameters:
1491
- {self.base_parameters}
1492
- transformations:
1493
- - type: AddFields
1494
- fields:
1495
- - path: ["field1"]
1496
- value: "1"
1497
- value_type: number
1498
- """
1499
- expected = [
1500
- AddFields(
1501
- fields=[
1502
- AddedFieldDefinition(
1503
- path=["field1"],
1504
- value=InterpolatedString(string="1", default="1", parameters={}),
1505
- value_type=float,
1506
- parameters={},
1507
- )
1508
- ],
1509
- parameters={},
1510
- )
1511
- ]
1512
- self._test_add_fields(content, expected)
1513
-
1514
- def test_add_fields_value_type_is_integer(self):
1515
- content = f"""
1516
- the_stream:
1517
- type: DeclarativeStream
1518
- $parameters:
1519
- {self.base_parameters}
1520
- transformations:
1521
- - type: AddFields
1522
- fields:
1523
- - path: ["field1"]
1524
- value: "1"
1525
- value_type: integer
1526
- """
1527
- expected = [
1528
- AddFields(
1529
- fields=[
1530
- AddedFieldDefinition(
1531
- path=["field1"],
1532
- value=InterpolatedString(string="1", default="1", parameters={}),
1533
- value_type=int,
1534
- parameters={},
1535
- )
1536
- ],
1537
- parameters={},
1538
- )
1539
- ]
1540
- self._test_add_fields(content, expected)
1541
-
1542
- def test_add_fields_value_type_is_boolean(self):
1543
- content = f"""
1544
- the_stream:
1545
- type: DeclarativeStream
1546
- $parameters:
1547
- {self.base_parameters}
1548
- transformations:
1549
- - type: AddFields
1550
- fields:
1551
- - path: ["field1"]
1552
- value: False
1553
- value_type: boolean
1554
- """
1555
- expected = [
1556
- AddFields(
1557
- fields=[
1558
- AddedFieldDefinition(
1559
- path=["field1"],
1560
- value=InterpolatedString(string="False", default="False", parameters={}),
1561
- value_type=bool,
1562
- parameters={},
1563
- )
1564
- ],
1565
- parameters={},
1566
- )
1567
- ]
1568
- self._test_add_fields(content, expected)
1569
-
1570
- def _test_add_fields(self, content, expected):
1571
- parsed_manifest = YamlDeclarativeSource._parse(content)
1572
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1573
- resolved_manifest["type"] = "DeclarativeSource"
1574
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["the_stream"], {})
1575
-
1576
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
1577
-
1578
- assert isinstance(stream, DeclarativeStream)
1579
- assert stream.retriever.record_selector.transformations == expected
1580
-
1581
- def test_default_schema_loader(self):
1582
- component_definition = {
1583
- "type": "DeclarativeStream",
1584
- "name": "test",
1585
- "primary_key": [],
1586
- "retriever": {
1587
- "type": "SimpleRetriever",
1588
- "requester": {
1589
- "type": "HttpRequester",
1590
- "url_base": "http://localhost:6767/",
1591
- "path": "items/",
1592
- "request_options_provider": {
1593
- "request_parameters": {},
1594
- "request_headers": {},
1595
- "request_body_json": {},
1596
- "type": "InterpolatedRequestOptionsProvider",
1597
- },
1598
- "authenticator": {"type": "BearerAuthenticator", "api_token": "{{ config['api_key'] }}"},
1599
- },
1600
- "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": ["items"]}},
1601
- "paginator": {"type": "NoPagination"},
1602
- },
1603
- }
1604
- resolved_manifest = resolver.preprocess_manifest(component_definition)
1605
- ws = ManifestComponentTransformer()
1606
- propagated_source_config = ws.propagate_types_and_parameters("", resolved_manifest, {})
1607
- stream = factory.create_component(
1608
- model_type=DeclarativeStreamModel, component_definition=propagated_source_config, config=input_config
1609
- )
1610
- schema_loader = stream.schema_loader
1611
- assert schema_loader.default_loader._get_json_filepath().split("/")[-1] == f"{stream.name}.json"
1612
-
1613
-
1614
- @pytest.mark.parametrize(
1615
- "incremental, partition_router, expected_type",
1616
- [
1617
- pytest.param(
1618
- {
1619
- "type": "DatetimeBasedCursor",
1620
- "datetime_format": "%Y-%m-%dT%H:%M:%S.%f%z",
1621
- "start_datetime": "{{ config['start_time'] }}",
1622
- "end_datetime": "{{ config['end_time'] }}",
1623
- "step": "P10D",
1624
- "cursor_field": "created",
1625
- "cursor_granularity": "PT0.000001S",
1626
- },
1627
- None,
1628
- DatetimeBasedCursor,
1629
- id="test_create_simple_retriever_with_incremental",
1630
- ),
1631
- pytest.param(
1632
- None,
1633
- {
1634
- "type": "ListPartitionRouter",
1635
- "values": "{{config['repos']}}",
1636
- "cursor_field": "a_key",
1637
- },
1638
- ListPartitionRouter,
1639
- id="test_create_simple_retriever_with_partition_router",
1640
- ),
1641
- pytest.param(
1642
- {
1643
- "type": "DatetimeBasedCursor",
1644
- "datetime_format": "%Y-%m-%dT%H:%M:%S.%f%z",
1645
- "start_datetime": "{{ config['start_time'] }}",
1646
- "end_datetime": "{{ config['end_time'] }}",
1647
- "step": "P10D",
1648
- "cursor_field": "created",
1649
- "cursor_granularity": "PT0.000001S",
1650
- },
1651
- {
1652
- "type": "ListPartitionRouter",
1653
- "values": "{{config['repos']}}",
1654
- "cursor_field": "a_key",
1655
- },
1656
- PerPartitionCursor,
1657
- id="test_create_simple_retriever_with_incremental_and_partition_router",
1658
- ),
1659
- pytest.param(
1660
- {
1661
- "type": "DatetimeBasedCursor",
1662
- "datetime_format": "%Y-%m-%dT%H:%M:%S.%f%z",
1663
- "start_datetime": "{{ config['start_time'] }}",
1664
- "end_datetime": "{{ config['end_time'] }}",
1665
- "step": "P10D",
1666
- "cursor_field": "created",
1667
- "cursor_granularity": "PT0.000001S",
1668
- },
1669
- [
1670
- {
1671
- "type": "ListPartitionRouter",
1672
- "values": "{{config['repos']}}",
1673
- "cursor_field": "a_key",
1674
- },
1675
- {
1676
- "type": "ListPartitionRouter",
1677
- "values": "{{config['repos']}}",
1678
- "cursor_field": "b_key",
1679
- },
1680
- ],
1681
- PerPartitionCursor,
1682
- id="test_create_simple_retriever_with_partition_routers_multiple_components",
1683
- ),
1684
- pytest.param(None, None, SinglePartitionRouter, id="test_create_simple_retriever_with_no_incremental_or_partition_router"),
1685
- ],
1686
- )
1687
- def test_merge_incremental_and_partition_router(incremental, partition_router, expected_type):
1688
- stream_model = {
1689
- "type": "DeclarativeStream",
1690
- "retriever": {
1691
- "type": "SimpleRetriever",
1692
- "record_selector": {
1693
- "type": "RecordSelector",
1694
- "extractor": {
1695
- "type": "DpathExtractor",
1696
- "field_path": [],
1697
- },
1698
- },
1699
- "requester": {
1700
- "type": "HttpRequester",
1701
- "name": "list",
1702
- "url_base": "orange.com",
1703
- "path": "/v1/api",
1704
- },
1705
- },
1706
- }
1707
-
1708
- if incremental:
1709
- stream_model["incremental_sync"] = incremental
1710
-
1711
- if partition_router:
1712
- stream_model["retriever"]["partition_router"] = partition_router
1713
-
1714
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_model, config=input_config)
1715
-
1716
- assert isinstance(stream, DeclarativeStream)
1717
- assert isinstance(stream.retriever, SimpleRetriever)
1718
- assert isinstance(stream.retriever.stream_slicer, expected_type)
1719
-
1720
- if incremental and partition_router:
1721
- assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor)
1722
- if type(partition_router) == list and len(partition_router) > 1:
1723
- assert type(stream.retriever.stream_slicer._partition_router) == CartesianProductStreamSlicer
1724
- assert len(stream.retriever.stream_slicer._partition_router.stream_slicers) == len(partition_router)
1725
- elif partition_router and type(partition_router) == list and len(partition_router) > 1:
1726
- assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor)
1727
- assert len(stream.retriever.stream_slicer.stream_slicerS) == len(partition_router)
1728
-
1729
-
1730
- def test_simple_retriever_emit_log_messages():
1731
- simple_retriever_model = {
1732
- "type": "SimpleRetriever",
1733
- "record_selector": {
1734
- "type": "RecordSelector",
1735
- "extractor": {
1736
- "type": "DpathExtractor",
1737
- "field_path": [],
1738
- },
1739
- },
1740
- "requester": {"type": "HttpRequester", "name": "list", "url_base": "orange.com", "path": "/v1/api"},
1741
- }
1742
-
1743
- connector_builder_factory = ModelToComponentFactory(emit_connector_builder_messages=True)
1744
- retriever = connector_builder_factory.create_component(
1745
- model_type=SimpleRetrieverModel,
1746
- component_definition=simple_retriever_model,
1747
- config={},
1748
- name="Test",
1749
- primary_key="id",
1750
- stream_slicer=None,
1751
- transformations=[],
1752
- )
1753
-
1754
- assert isinstance(retriever, SimpleRetrieverTestReadDecorator)
1755
- assert connector_builder_factory._message_repository._log_level == Level.DEBUG
1756
-
1757
-
1758
- def test_ignore_retry():
1759
- requester_model = {
1760
- "type": "HttpRequester",
1761
- "name": "list",
1762
- "url_base": "orange.com",
1763
- "path": "/v1/api",
1764
- }
1765
-
1766
- connector_builder_factory = ModelToComponentFactory(disable_retries=True)
1767
- requester = connector_builder_factory.create_component(
1768
- model_type=HttpRequesterModel,
1769
- component_definition=requester_model,
1770
- config={},
1771
- name="Test",
1772
- )
1773
-
1774
- assert requester.max_retries == 0
1775
-
1776
-
1777
- def test_create_page_increment():
1778
- model = PageIncrementModel(
1779
- type="PageIncrement",
1780
- page_size=10,
1781
- start_from_page=1,
1782
- inject_on_first_request=True,
1783
- )
1784
- expected_strategy = PageIncrement(page_size=10, start_from_page=1, inject_on_first_request=True, parameters={}, config=input_config)
1785
-
1786
- strategy = factory.create_page_increment(model, input_config)
1787
-
1788
- assert strategy.page_size == expected_strategy.page_size
1789
- assert strategy.start_from_page == expected_strategy.start_from_page
1790
- assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request
1791
-
1792
-
1793
- def test_create_page_increment_with_interpolated_page_size():
1794
- model = PageIncrementModel(
1795
- type="PageIncrement",
1796
- page_size="{{ config['page_size'] }}",
1797
- start_from_page=1,
1798
- inject_on_first_request=True,
1799
- )
1800
- config = {
1801
- **input_config,
1802
- "page_size": 5
1803
- }
1804
- expected_strategy = PageIncrement(page_size=5, start_from_page=1, inject_on_first_request=True, parameters={}, config=config)
1805
-
1806
- strategy = factory.create_page_increment(model, config)
1807
-
1808
- assert strategy.get_page_size() == expected_strategy.get_page_size()
1809
- assert strategy.start_from_page == expected_strategy.start_from_page
1810
- assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request
1811
-
1812
-
1813
- def test_create_offset_increment():
1814
- model = OffsetIncrementModel(
1815
- type="OffsetIncrement",
1816
- page_size=10,
1817
- inject_on_first_request=True,
1818
- )
1819
- expected_strategy = OffsetIncrement(page_size=10, inject_on_first_request=True, parameters={}, config=input_config)
1820
-
1821
- strategy = factory.create_offset_increment(model, input_config)
1822
-
1823
- assert strategy.page_size == expected_strategy.page_size
1824
- assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request
1825
- assert strategy.config == input_config
1826
-
1827
-
1828
- class MyCustomSchemaLoader(SchemaLoader):
1829
- def get_json_schema(self) -> Mapping[str, Any]:
1830
- """Returns a mapping describing the stream's schema"""
1831
- return {}
1832
-
1833
-
1834
- def test_create_custom_schema_loader():
1835
-
1836
- definition = {
1837
- "type": "CustomSchemaLoader",
1838
- "class_name": "unit_tests.sources.declarative.parsers.test_model_to_component_factory.MyCustomSchemaLoader"
1839
- }
1840
- component = factory.create_component(CustomSchemaLoaderModel, definition, {})
1841
- assert isinstance(component, MyCustomSchemaLoader)