airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (518) hide show
  1. airbyte_cdk/__init__.py +355 -6
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +29 -10
  7. airbyte_cdk/connector.py +24 -24
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
  10. airbyte_cdk/connector_builder/main.py +45 -13
  11. airbyte_cdk/connector_builder/message_grouper.py +189 -50
  12. airbyte_cdk/connector_builder/models.py +3 -2
  13. airbyte_cdk/destinations/__init__.py +4 -3
  14. airbyte_cdk/destinations/destination.py +54 -20
  15. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  16. airbyte_cdk/destinations/vector_db_based/config.py +40 -17
  17. airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
  18. airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
  19. airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
  20. airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
  21. airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
  22. airbyte_cdk/entrypoint.py +153 -44
  23. airbyte_cdk/exception_handler.py +21 -3
  24. airbyte_cdk/logger.py +30 -44
  25. airbyte_cdk/models/__init__.py +13 -2
  26. airbyte_cdk/models/airbyte_protocol.py +86 -1
  27. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  28. airbyte_cdk/models/file_transfer_record_message.py +13 -0
  29. airbyte_cdk/models/well_known_types.py +1 -1
  30. airbyte_cdk/sources/__init__.py +5 -1
  31. airbyte_cdk/sources/abstract_source.py +125 -79
  32. airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
  33. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
  34. airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
  35. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
  36. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  37. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
  38. airbyte_cdk/sources/config.py +3 -2
  39. airbyte_cdk/sources/connector_state_manager.py +49 -83
  40. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  41. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
  42. airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
  43. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  44. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  45. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  46. airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
  47. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
  48. airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
  49. airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
  50. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
  51. airbyte_cdk/sources/declarative/auth/token.py +28 -10
  52. airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
  53. airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
  54. airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
  55. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  56. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  57. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
  58. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
  59. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
  60. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1185 -85
  61. airbyte_cdk/sources/declarative/declarative_source.py +5 -2
  62. airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
  63. airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
  64. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
  65. airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
  66. airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
  67. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  68. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  69. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  70. airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
  71. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
  72. airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
  73. airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
  74. airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
  75. airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
  76. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
  77. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  78. airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
  79. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +346 -0
  80. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
  81. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  82. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
  83. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +173 -74
  84. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  85. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  86. airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
  87. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
  88. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
  89. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
  90. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
  91. airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
  92. airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
  93. airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
  94. airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
  95. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  96. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  97. airbyte_cdk/sources/declarative/models/__init__.py +1 -1
  98. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1319 -603
  99. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
  100. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
  101. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
  102. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1759 -225
  103. airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
  104. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  105. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  106. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
  107. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  108. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
  109. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
  110. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
  111. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
  112. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
  113. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
  114. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
  115. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
  116. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
  117. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
  118. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
  119. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
  120. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  121. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
  122. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
  123. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
  124. airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
  125. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
  126. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
  127. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
  128. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
  129. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
  130. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
  131. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
  132. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
  133. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
  134. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
  135. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
  136. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
  137. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  138. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
  139. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
  140. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
  141. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
  142. airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
  143. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  144. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  145. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  146. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  147. airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
  148. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
  149. airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
  150. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
  151. airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
  152. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
  153. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
  154. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
  155. airbyte_cdk/sources/declarative/spec/spec.py +12 -5
  156. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
  157. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
  158. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
  159. airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
  160. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  161. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  162. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  163. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  164. airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
  165. airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
  166. airbyte_cdk/sources/declarative/types.py +19 -110
  167. airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
  168. airbyte_cdk/sources/embedded/base_integration.py +16 -5
  169. airbyte_cdk/sources/embedded/catalog.py +16 -4
  170. airbyte_cdk/sources/embedded/runner.py +19 -3
  171. airbyte_cdk/sources/embedded/tools.py +5 -2
  172. airbyte_cdk/sources/file_based/README.md +152 -0
  173. airbyte_cdk/sources/file_based/__init__.py +24 -0
  174. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
  175. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
  176. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
  177. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
  178. airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
  179. airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
  180. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  181. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
  182. airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
  183. airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
  184. airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
  185. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
  186. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
  187. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
  188. airbyte_cdk/sources/file_based/exceptions.py +18 -15
  189. airbyte_cdk/sources/file_based/file_based_source.py +140 -33
  190. airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
  191. airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
  192. airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
  193. airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
  194. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  195. airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
  196. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
  197. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
  198. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
  199. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
  200. airbyte_cdk/sources/file_based/remote_file.py +1 -1
  201. airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
  202. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
  203. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
  204. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
  205. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
  206. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
  207. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
  208. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
  209. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
  210. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
  211. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
  212. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
  213. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
  214. airbyte_cdk/sources/http_logger.py +8 -3
  215. airbyte_cdk/sources/message/__init__.py +7 -1
  216. airbyte_cdk/sources/message/repository.py +18 -4
  217. airbyte_cdk/sources/source.py +42 -38
  218. airbyte_cdk/sources/streams/__init__.py +2 -2
  219. airbyte_cdk/sources/streams/availability_strategy.py +54 -3
  220. airbyte_cdk/sources/streams/call_rate.py +64 -21
  221. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  222. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  223. airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
  224. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  225. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  226. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  227. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  228. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
  229. airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
  230. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
  231. airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
  232. airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
  233. airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
  234. airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
  235. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
  236. airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
  237. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
  238. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  239. airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
  240. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
  241. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
  242. airbyte_cdk/sources/streams/core.py +412 -87
  243. airbyte_cdk/sources/streams/http/__init__.py +2 -1
  244. airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
  245. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  246. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  247. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  248. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  249. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  250. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  251. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  252. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  253. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  254. airbyte_cdk/sources/streams/http/exceptions.py +27 -7
  255. airbyte_cdk/sources/streams/http/http.py +369 -246
  256. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  257. airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
  258. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
  259. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
  260. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
  261. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
  262. airbyte_cdk/sources/types.py +154 -0
  263. airbyte_cdk/sources/utils/record_helper.py +36 -21
  264. airbyte_cdk/sources/utils/schema_helpers.py +13 -6
  265. airbyte_cdk/sources/utils/slice_logger.py +4 -1
  266. airbyte_cdk/sources/utils/transform.py +54 -20
  267. airbyte_cdk/sql/_util/hashing.py +34 -0
  268. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  269. airbyte_cdk/sql/constants.py +32 -0
  270. airbyte_cdk/sql/exceptions.py +235 -0
  271. airbyte_cdk/sql/secrets.py +123 -0
  272. airbyte_cdk/sql/shared/__init__.py +15 -0
  273. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  274. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  275. airbyte_cdk/sql/types.py +160 -0
  276. airbyte_cdk/test/catalog_builder.py +70 -18
  277. airbyte_cdk/test/entrypoint_wrapper.py +117 -42
  278. airbyte_cdk/test/mock_http/__init__.py +1 -1
  279. airbyte_cdk/test/mock_http/matcher.py +6 -0
  280. airbyte_cdk/test/mock_http/mocker.py +57 -10
  281. airbyte_cdk/test/mock_http/request.py +19 -3
  282. airbyte_cdk/test/mock_http/response.py +3 -1
  283. airbyte_cdk/test/mock_http/response_builder.py +32 -16
  284. airbyte_cdk/test/state_builder.py +18 -10
  285. airbyte_cdk/test/utils/__init__.py +1 -0
  286. airbyte_cdk/test/utils/data.py +24 -0
  287. airbyte_cdk/test/utils/http_mocking.py +16 -0
  288. airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
  289. airbyte_cdk/test/utils/reading.py +26 -0
  290. airbyte_cdk/utils/__init__.py +2 -1
  291. airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
  292. airbyte_cdk/utils/analytics_message.py +10 -2
  293. airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
  294. airbyte_cdk/utils/event_timing.py +10 -10
  295. airbyte_cdk/utils/mapping_helpers.py +3 -1
  296. airbyte_cdk/utils/message_utils.py +20 -11
  297. airbyte_cdk/utils/print_buffer.py +75 -0
  298. airbyte_cdk/utils/schema_inferrer.py +198 -28
  299. airbyte_cdk/utils/slice_hasher.py +30 -0
  300. airbyte_cdk/utils/spec_schema_transformations.py +6 -3
  301. airbyte_cdk/utils/stream_status_utils.py +8 -1
  302. airbyte_cdk/utils/traced_exception.py +61 -21
  303. airbyte_cdk-6.17.1.dev0.dist-info/METADATA +109 -0
  304. airbyte_cdk-6.17.1.dev0.dist-info/RECORD +350 -0
  305. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/WHEEL +1 -2
  306. airbyte_cdk-6.17.1.dev0.dist-info/entry_points.txt +3 -0
  307. airbyte_cdk/sources/declarative/create_partial.py +0 -92
  308. airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
  309. airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
  310. airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
  311. airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
  312. airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
  313. airbyte_cdk/sources/deprecated/base_source.py +0 -94
  314. airbyte_cdk/sources/deprecated/client.py +0 -99
  315. airbyte_cdk/sources/singer/__init__.py +0 -8
  316. airbyte_cdk/sources/singer/singer_helpers.py +0 -304
  317. airbyte_cdk/sources/singer/source.py +0 -186
  318. airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
  319. airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
  320. airbyte_cdk/sources/streams/http/auth/core.py +0 -29
  321. airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
  322. airbyte_cdk/sources/streams/http/auth/token.py +0 -47
  323. airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
  324. airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
  325. airbyte_cdk/sources/utils/schema_models.py +0 -84
  326. airbyte_cdk-0.72.1.dist-info/METADATA +0 -243
  327. airbyte_cdk-0.72.1.dist-info/RECORD +0 -466
  328. airbyte_cdk-0.72.1.dist-info/top_level.txt +0 -3
  329. source_declarative_manifest/main.py +0 -29
  330. unit_tests/connector_builder/__init__.py +0 -3
  331. unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
  332. unit_tests/connector_builder/test_message_grouper.py +0 -713
  333. unit_tests/connector_builder/utils.py +0 -27
  334. unit_tests/destinations/test_destination.py +0 -243
  335. unit_tests/singer/test_singer_helpers.py +0 -56
  336. unit_tests/singer/test_singer_source.py +0 -112
  337. unit_tests/sources/__init__.py +0 -0
  338. unit_tests/sources/concurrent_source/__init__.py +0 -3
  339. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
  340. unit_tests/sources/declarative/__init__.py +0 -3
  341. unit_tests/sources/declarative/auth/__init__.py +0 -3
  342. unit_tests/sources/declarative/auth/test_oauth.py +0 -331
  343. unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
  344. unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
  345. unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
  346. unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
  347. unit_tests/sources/declarative/checks/__init__.py +0 -3
  348. unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
  349. unit_tests/sources/declarative/decoders/__init__.py +0 -0
  350. unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
  351. unit_tests/sources/declarative/external_component.py +0 -13
  352. unit_tests/sources/declarative/extractors/__init__.py +0 -3
  353. unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
  354. unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
  355. unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
  356. unit_tests/sources/declarative/incremental/__init__.py +0 -0
  357. unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
  358. unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
  359. unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
  360. unit_tests/sources/declarative/interpolation/__init__.py +0 -3
  361. unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
  362. unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
  363. unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
  364. unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
  365. unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
  366. unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
  367. unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
  368. unit_tests/sources/declarative/parsers/__init__.py +0 -3
  369. unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
  370. unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
  371. unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1847
  372. unit_tests/sources/declarative/parsers/testing_components.py +0 -36
  373. unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
  374. unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
  375. unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
  376. unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
  377. unit_tests/sources/declarative/requesters/__init__.py +0 -3
  378. unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
  379. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
  380. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
  381. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
  382. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
  383. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
  384. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
  385. unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
  386. unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
  387. unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
  388. unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
  389. unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
  390. unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
  391. unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
  392. unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
  393. unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
  394. unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
  395. unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
  396. unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
  397. unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
  398. unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
  399. unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
  400. unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
  401. unit_tests/sources/declarative/retrievers/__init__.py +0 -3
  402. unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
  403. unit_tests/sources/declarative/schema/__init__.py +0 -6
  404. unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
  405. unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
  406. unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
  407. unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
  408. unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
  409. unit_tests/sources/declarative/states/__init__.py +0 -3
  410. unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
  411. unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
  412. unit_tests/sources/declarative/test_create_partial.py +0 -83
  413. unit_tests/sources/declarative/test_declarative_stream.py +0 -103
  414. unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
  415. unit_tests/sources/declarative/test_types.py +0 -39
  416. unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
  417. unit_tests/sources/file_based/__init__.py +0 -0
  418. unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
  419. unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
  420. unit_tests/sources/file_based/config/__init__.py +0 -0
  421. unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
  422. unit_tests/sources/file_based/config/test_csv_format.py +0 -34
  423. unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
  424. unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
  425. unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
  426. unit_tests/sources/file_based/file_types/__init__.py +0 -0
  427. unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
  428. unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
  429. unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
  430. unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
  431. unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
  432. unit_tests/sources/file_based/helpers.py +0 -70
  433. unit_tests/sources/file_based/in_memory_files_source.py +0 -211
  434. unit_tests/sources/file_based/scenarios/__init__.py +0 -0
  435. unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
  436. unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
  437. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
  438. unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
  439. unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
  440. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
  441. unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
  442. unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
  443. unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
  444. unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
  445. unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
  446. unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
  447. unit_tests/sources/file_based/stream/__init__.py +0 -0
  448. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  449. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
  450. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
  451. unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
  452. unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
  453. unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
  454. unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
  455. unit_tests/sources/file_based/test_scenarios.py +0 -253
  456. unit_tests/sources/file_based/test_schema_helpers.py +0 -346
  457. unit_tests/sources/fixtures/__init__.py +0 -3
  458. unit_tests/sources/fixtures/source_test_fixture.py +0 -153
  459. unit_tests/sources/message/__init__.py +0 -0
  460. unit_tests/sources/message/test_repository.py +0 -153
  461. unit_tests/sources/streams/__init__.py +0 -0
  462. unit_tests/sources/streams/concurrent/__init__.py +0 -3
  463. unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
  464. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
  465. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
  466. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
  467. unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
  468. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
  469. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
  470. unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
  471. unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
  472. unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
  473. unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
  474. unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
  475. unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
  476. unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
  477. unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
  478. unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
  479. unit_tests/sources/streams/http/__init__.py +0 -0
  480. unit_tests/sources/streams/http/auth/__init__.py +0 -0
  481. unit_tests/sources/streams/http/auth/test_auth.py +0 -173
  482. unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
  483. unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
  484. unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
  485. unit_tests/sources/streams/http/test_http.py +0 -635
  486. unit_tests/sources/streams/test_availability_strategy.py +0 -70
  487. unit_tests/sources/streams/test_call_rate.py +0 -300
  488. unit_tests/sources/streams/test_stream_read.py +0 -405
  489. unit_tests/sources/streams/test_streams_core.py +0 -184
  490. unit_tests/sources/test_abstract_source.py +0 -1442
  491. unit_tests/sources/test_concurrent_source.py +0 -112
  492. unit_tests/sources/test_config.py +0 -92
  493. unit_tests/sources/test_connector_state_manager.py +0 -482
  494. unit_tests/sources/test_http_logger.py +0 -252
  495. unit_tests/sources/test_integration_source.py +0 -86
  496. unit_tests/sources/test_source.py +0 -684
  497. unit_tests/sources/test_source_read.py +0 -460
  498. unit_tests/test/__init__.py +0 -0
  499. unit_tests/test/mock_http/__init__.py +0 -0
  500. unit_tests/test/mock_http/test_matcher.py +0 -53
  501. unit_tests/test/mock_http/test_mocker.py +0 -214
  502. unit_tests/test/mock_http/test_request.py +0 -117
  503. unit_tests/test/mock_http/test_response_builder.py +0 -177
  504. unit_tests/test/test_entrypoint_wrapper.py +0 -240
  505. unit_tests/utils/__init__.py +0 -0
  506. unit_tests/utils/test_datetime_format_inferrer.py +0 -60
  507. unit_tests/utils/test_mapping_helpers.py +0 -54
  508. unit_tests/utils/test_message_utils.py +0 -91
  509. unit_tests/utils/test_rate_limiting.py +0 -26
  510. unit_tests/utils/test_schema_inferrer.py +0 -202
  511. unit_tests/utils/test_secret_utils.py +0 -135
  512. unit_tests/utils/test_stream_status_utils.py +0 -61
  513. unit_tests/utils/test_traced_exception.py +0 -107
  514. /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
  515. {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
  516. {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
  517. {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
  518. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/LICENSE.txt +0 -0
@@ -1,1847 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- # mypy: ignore-errors
6
-
7
- import datetime
8
- from typing import Any, Mapping
9
-
10
- import pytest
11
- from airbyte_cdk.models import Level
12
- from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator
13
- from airbyte_cdk.sources.declarative.auth.token import (
14
- ApiKeyAuthenticator,
15
- BasicHttpAuthenticator,
16
- BearerAuthenticator,
17
- LegacySessionTokenAuthenticator,
18
- )
19
- from airbyte_cdk.sources.declarative.auth.token_provider import SessionTokenProvider
20
- from airbyte_cdk.sources.declarative.checks import CheckStream
21
- from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime
22
- from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
23
- from airbyte_cdk.sources.declarative.decoders import JsonDecoder
24
- from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordFilter, RecordSelector
25
- from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor, PerPartitionCursor
26
- from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
27
- from airbyte_cdk.sources.declarative.models import CheckStream as CheckStreamModel
28
- from airbyte_cdk.sources.declarative.models import CompositeErrorHandler as CompositeErrorHandlerModel
29
- from airbyte_cdk.sources.declarative.models import CustomErrorHandler as CustomErrorHandlerModel
30
- from airbyte_cdk.sources.declarative.models import CustomPartitionRouter as CustomPartitionRouterModel
31
- from airbyte_cdk.sources.declarative.models import CustomSchemaLoader as CustomSchemaLoaderModel
32
- from airbyte_cdk.sources.declarative.models import DatetimeBasedCursor as DatetimeBasedCursorModel
33
- from airbyte_cdk.sources.declarative.models import DeclarativeStream as DeclarativeStreamModel
34
- from airbyte_cdk.sources.declarative.models import DefaultPaginator as DefaultPaginatorModel
35
- from airbyte_cdk.sources.declarative.models import HttpRequester as HttpRequesterModel
36
- from airbyte_cdk.sources.declarative.models import ListPartitionRouter as ListPartitionRouterModel
37
- from airbyte_cdk.sources.declarative.models import OAuthAuthenticator as OAuthAuthenticatorModel
38
- from airbyte_cdk.sources.declarative.models import RecordSelector as RecordSelectorModel
39
- from airbyte_cdk.sources.declarative.models import SimpleRetriever as SimpleRetrieverModel
40
- from airbyte_cdk.sources.declarative.models import Spec as SpecModel
41
- from airbyte_cdk.sources.declarative.models import SubstreamPartitionRouter as SubstreamPartitionRouterModel
42
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import OffsetIncrement as OffsetIncrementModel
43
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import PageIncrement as PageIncrementModel
44
- from airbyte_cdk.sources.declarative.models.declarative_component_schema import SelectiveAuthenticator
45
- from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer
46
- from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver
47
- from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory
48
- from airbyte_cdk.sources.declarative.partition_routers import ListPartitionRouter, SinglePartitionRouter, SubstreamPartitionRouter
49
- from airbyte_cdk.sources.declarative.requesters import HttpRequester
50
- from airbyte_cdk.sources.declarative.requesters.error_handlers import CompositeErrorHandler, DefaultErrorHandler, HttpResponseFilter
51
- from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import (
52
- ConstantBackoffStrategy,
53
- ExponentialBackoffStrategy,
54
- WaitTimeFromHeaderBackoffStrategy,
55
- WaitUntilTimeFromHeaderBackoffStrategy,
56
- )
57
- from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction
58
- from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator
59
- from airbyte_cdk.sources.declarative.requesters.paginators.strategies import (
60
- CursorPaginationStrategy,
61
- OffsetIncrement,
62
- PageIncrement,
63
- StopConditionPaginationStrategyDecorator,
64
- )
65
- from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType
66
- from airbyte_cdk.sources.declarative.requesters.request_options import InterpolatedRequestOptionsProvider
67
- from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
68
- from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
69
- from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever, SimpleRetrieverTestReadDecorator
70
- from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader
71
- from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
72
- from airbyte_cdk.sources.declarative.spec import Spec
73
- from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStreamSlicer
74
- from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields
75
- from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
76
- from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
77
- from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator
78
- from unit_tests.sources.declarative.parsers.testing_components import TestingCustomSubstreamPartitionRouter, TestingSomeComponent
79
-
80
- factory = ModelToComponentFactory()
81
-
82
- resolver = ManifestReferenceResolver()
83
-
84
- transformer = ManifestComponentTransformer()
85
-
86
- input_config = {"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}
87
-
88
-
89
- def test_create_check_stream():
90
- manifest = {"check": {"type": "CheckStream", "stream_names": ["list_stream"]}}
91
-
92
- check = factory.create_component(CheckStreamModel, manifest["check"], {})
93
-
94
- assert isinstance(check, CheckStream)
95
- assert check.stream_names == ["list_stream"]
96
-
97
-
98
- def test_create_component_type_mismatch():
99
- manifest = {"check": {"type": "MismatchType", "stream_names": ["list_stream"]}}
100
-
101
- with pytest.raises(ValueError):
102
- factory.create_component(CheckStreamModel, manifest["check"], {})
103
-
104
-
105
- def test_full_config_stream():
106
- content = """
107
- decoder:
108
- type: JsonDecoder
109
- extractor:
110
- type: DpathExtractor
111
- decoder: "#/decoder"
112
- selector:
113
- type: RecordSelector
114
- record_filter:
115
- type: RecordFilter
116
- condition: "{{ record['id'] > stream_state['id'] }}"
117
- metadata_paginator:
118
- type: DefaultPaginator
119
- page_size_option:
120
- type: RequestOption
121
- inject_into: request_parameter
122
- field_name: page_size
123
- page_token_option:
124
- type: RequestPath
125
- pagination_strategy:
126
- type: "CursorPagination"
127
- cursor_value: "{{ response._metadata.next }}"
128
- page_size: 10
129
- requester:
130
- type: HttpRequester
131
- url_base: "https://api.sendgrid.com/v3/"
132
- http_method: "GET"
133
- authenticator:
134
- type: BearerAuthenticator
135
- api_token: "{{ config['apikey'] }}"
136
- request_parameters:
137
- unit: "day"
138
- retriever:
139
- paginator:
140
- type: NoPagination
141
- partial_stream:
142
- type: DeclarativeStream
143
- schema_loader:
144
- type: JsonFileSchemaLoader
145
- file_path: "./source_sendgrid/schemas/{{ parameters.name }}.json"
146
- list_stream:
147
- $ref: "#/partial_stream"
148
- $parameters:
149
- name: "lists"
150
- extractor:
151
- $ref: "#/extractor"
152
- field_path: ["{{ parameters['name'] }}"]
153
- name: "lists"
154
- primary_key: "id"
155
- retriever:
156
- $ref: "#/retriever"
157
- requester:
158
- $ref: "#/requester"
159
- path: "{{ next_page_token['next_page_url'] }}"
160
- paginator:
161
- $ref: "#/metadata_paginator"
162
- record_selector:
163
- $ref: "#/selector"
164
- transformations:
165
- - type: AddFields
166
- fields:
167
- - path: ["extra"]
168
- value: "{{ response.to_add }}"
169
- incremental_sync:
170
- type: DatetimeBasedCursor
171
- start_datetime: "{{ config['start_time'] }}"
172
- end_datetime: "{{ config['end_time'] }}"
173
- step: "P10D"
174
- cursor_field: "created"
175
- cursor_granularity: "PT0.000001S"
176
- $parameters:
177
- datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
178
- check:
179
- type: CheckStream
180
- stream_names: ["list_stream"]
181
- spec:
182
- type: Spec
183
- documentation_url: https://airbyte.com/#yaml-from-manifest
184
- connection_specification:
185
- title: Test Spec
186
- type: object
187
- required:
188
- - api_key
189
- additionalProperties: false
190
- properties:
191
- api_key:
192
- type: string
193
- airbyte_secret: true
194
- title: API Key
195
- description: Test API Key
196
- order: 0
197
- advanced_auth:
198
- auth_flow_type: "oauth2.0"
199
- """
200
- parsed_manifest = YamlDeclarativeSource._parse(content)
201
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
202
- resolved_manifest["type"] = "DeclarativeSource"
203
- manifest = transformer.propagate_types_and_parameters("", resolved_manifest, {})
204
-
205
- stream_manifest = manifest["list_stream"]
206
- assert stream_manifest["type"] == "DeclarativeStream"
207
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
208
-
209
- assert isinstance(stream, DeclarativeStream)
210
- assert stream.primary_key == "id"
211
- assert stream.name == "lists"
212
- assert stream._stream_cursor_field.string == "created"
213
-
214
- assert isinstance(stream.schema_loader, JsonFileSchemaLoader)
215
- assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.json"
216
-
217
- assert len(stream.retriever.record_selector.transformations) == 1
218
- add_fields = stream.retriever.record_selector.transformations[0]
219
- assert isinstance(add_fields, AddFields)
220
- assert add_fields.fields[0].path == ["extra"]
221
- assert add_fields.fields[0].value.string == "{{ response.to_add }}"
222
-
223
- assert isinstance(stream.retriever, SimpleRetriever)
224
- assert stream.retriever.primary_key == stream.primary_key
225
- assert stream.retriever.name == stream.name
226
-
227
- assert isinstance(stream.retriever.record_selector, RecordSelector)
228
-
229
- assert isinstance(stream.retriever.record_selector.extractor, DpathExtractor)
230
- assert isinstance(stream.retriever.record_selector.extractor.decoder, JsonDecoder)
231
- assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_path] == ["lists"]
232
-
233
- assert isinstance(stream.retriever.record_selector.record_filter, RecordFilter)
234
- assert stream.retriever.record_selector.record_filter._filter_interpolator.condition == "{{ record['id'] > stream_state['id'] }}"
235
-
236
- assert isinstance(stream.retriever.paginator, DefaultPaginator)
237
- assert isinstance(stream.retriever.paginator.decoder, JsonDecoder)
238
- assert stream.retriever.paginator.page_size_option.field_name.eval(input_config) == "page_size"
239
- assert stream.retriever.paginator.page_size_option.inject_into == RequestOptionType.request_parameter
240
- assert isinstance(stream.retriever.paginator.page_token_option, RequestPath)
241
- assert stream.retriever.paginator.url_base.string == "https://api.sendgrid.com/v3/"
242
- assert stream.retriever.paginator.url_base.default == "https://api.sendgrid.com/v3/"
243
-
244
- assert isinstance(stream.retriever.paginator.pagination_strategy, CursorPaginationStrategy)
245
- assert isinstance(stream.retriever.paginator.pagination_strategy.decoder, JsonDecoder)
246
- assert stream.retriever.paginator.pagination_strategy.cursor_value.string == "{{ response._metadata.next }}"
247
- assert stream.retriever.paginator.pagination_strategy.cursor_value.default == "{{ response._metadata.next }}"
248
- assert stream.retriever.paginator.pagination_strategy.page_size == 10
249
-
250
- assert isinstance(stream.retriever.requester, HttpRequester)
251
- assert stream.retriever.requester.http_method == HttpMethod.GET
252
- assert stream.retriever.requester.name == stream.name
253
- assert stream.retriever.requester._path.string == "{{ next_page_token['next_page_url'] }}"
254
- assert stream.retriever.requester._path.default == "{{ next_page_token['next_page_url'] }}"
255
-
256
- assert isinstance(stream.retriever.requester.authenticator, BearerAuthenticator)
257
- assert stream.retriever.requester.authenticator.token_provider.get_token() == "verysecrettoken"
258
-
259
- assert isinstance(stream.retriever.requester.request_options_provider, InterpolatedRequestOptionsProvider)
260
- assert stream.retriever.requester.request_options_provider.request_parameters.get("unit") == "day"
261
-
262
- checker = factory.create_component(model_type=CheckStreamModel, component_definition=manifest["check"], config=input_config)
263
-
264
- assert isinstance(checker, CheckStream)
265
- streams_to_check = checker.stream_names
266
- assert len(streams_to_check) == 1
267
- assert list(streams_to_check)[0] == "list_stream"
268
-
269
- spec = factory.create_component(model_type=SpecModel, component_definition=manifest["spec"], config=input_config)
270
-
271
- assert isinstance(spec, Spec)
272
- documentation_url = spec.documentation_url
273
- connection_specification = spec.connection_specification
274
- assert documentation_url == "https://airbyte.com/#yaml-from-manifest"
275
- assert connection_specification["title"] == "Test Spec"
276
- assert connection_specification["required"] == ["api_key"]
277
- assert connection_specification["properties"]["api_key"] == {
278
- "type": "string",
279
- "airbyte_secret": True,
280
- "title": "API Key",
281
- "description": "Test API Key",
282
- "order": 0,
283
- }
284
- advanced_auth = spec.advanced_auth
285
- assert advanced_auth.auth_flow_type.value == "oauth2.0"
286
-
287
-
288
- def test_interpolate_config():
289
- content = """
290
- authenticator:
291
- type: OAuthAuthenticator
292
- client_id: "some_client_id"
293
- client_secret: "some_client_secret"
294
- token_refresh_endpoint: "https://api.sendgrid.com/v3/auth"
295
- refresh_token: "{{ config['apikey'] }}"
296
- refresh_request_body:
297
- body_field: "yoyoyo"
298
- interpolated_body_field: "{{ config['apikey'] }}"
299
- """
300
- parsed_manifest = YamlDeclarativeSource._parse(content)
301
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
302
- authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
303
-
304
- authenticator = factory.create_component(
305
- model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=input_config
306
- )
307
-
308
- assert isinstance(authenticator, DeclarativeOauth2Authenticator)
309
- assert authenticator._client_id.eval(input_config) == "some_client_id"
310
- assert authenticator._client_secret.string == "some_client_secret"
311
- assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth"
312
- assert authenticator._refresh_token.eval(input_config) == "verysecrettoken"
313
- assert authenticator._refresh_request_body.mapping == {"body_field": "yoyoyo", "interpolated_body_field": "{{ config['apikey'] }}"}
314
- assert authenticator.get_refresh_request_body() == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"}
315
-
316
-
317
- def test_interpolate_config_with_token_expiry_date_format():
318
- content = """
319
- authenticator:
320
- type: OAuthAuthenticator
321
- client_id: "some_client_id"
322
- client_secret: "some_client_secret"
323
- token_refresh_endpoint: "https://api.sendgrid.com/v3/auth"
324
- refresh_token: "{{ config['apikey'] }}"
325
- token_expiry_date_format: "%Y-%m-%d %H:%M:%S.%f+00:00"
326
- """
327
- parsed_manifest = YamlDeclarativeSource._parse(content)
328
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
329
- authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
330
-
331
- authenticator = factory.create_component(
332
- model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=input_config
333
- )
334
-
335
- assert isinstance(authenticator, DeclarativeOauth2Authenticator)
336
- assert authenticator.token_expiry_date_format == "%Y-%m-%d %H:%M:%S.%f+00:00"
337
- assert authenticator.token_expiry_is_time_of_expiration
338
- assert authenticator._client_id.eval(input_config) == "some_client_id"
339
- assert authenticator._client_secret.string == "some_client_secret"
340
- assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth"
341
-
342
-
343
- def test_single_use_oauth_branch():
344
- single_use_input_config = {
345
- "apikey": "verysecrettoken",
346
- "repos": ["airbyte", "airbyte-cloud"],
347
- "credentials": {"access_token": "access_token", "token_expiry_date": "1970-01-01"},
348
- }
349
-
350
- content = """
351
- authenticator:
352
- type: OAuthAuthenticator
353
- client_id: "some_client_id"
354
- client_secret: "some_client_secret"
355
- token_refresh_endpoint: "https://api.sendgrid.com/v3/auth"
356
- refresh_token: "{{ config['apikey'] }}"
357
- refresh_request_body:
358
- body_field: "yoyoyo"
359
- interpolated_body_field: "{{ config['apikey'] }}"
360
- refresh_token_updater:
361
- refresh_token_name: "the_refresh_token"
362
- refresh_token_error_status_codes: [400]
363
- refresh_token_error_key: "error"
364
- refresh_token_error_values: ["invalid_grant"]
365
- refresh_token_config_path:
366
- - apikey
367
- """
368
- parsed_manifest = YamlDeclarativeSource._parse(content)
369
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
370
- authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
371
-
372
- authenticator: SingleUseRefreshTokenOauth2Authenticator = factory.create_component(
373
- model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=single_use_input_config
374
- )
375
-
376
- assert isinstance(authenticator, SingleUseRefreshTokenOauth2Authenticator)
377
- assert authenticator._client_id == "some_client_id"
378
- assert authenticator._client_secret == "some_client_secret"
379
- assert authenticator._token_refresh_endpoint == "https://api.sendgrid.com/v3/auth"
380
- assert authenticator._refresh_token == "verysecrettoken"
381
- assert authenticator._refresh_request_body == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"}
382
- assert authenticator._refresh_token_name == "the_refresh_token"
383
- assert authenticator._refresh_token_config_path == ["apikey"]
384
- # default values
385
- assert authenticator._access_token_config_path == ["credentials", "access_token"]
386
- assert authenticator._token_expiry_date_config_path == ["credentials", "token_expiry_date"]
387
- assert authenticator._refresh_token_error_status_codes == [400]
388
- assert authenticator._refresh_token_error_key == "error"
389
- assert authenticator._refresh_token_error_values == ["invalid_grant"]
390
-
391
-
392
- def test_list_based_stream_slicer_with_values_refd():
393
- content = """
394
- repositories: ["airbyte", "airbyte-cloud"]
395
- partition_router:
396
- type: ListPartitionRouter
397
- values: "#/repositories"
398
- cursor_field: repository
399
- """
400
- parsed_manifest = YamlDeclarativeSource._parse(content)
401
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
402
- partition_router_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["partition_router"], {})
403
-
404
- partition_router = factory.create_component(
405
- model_type=ListPartitionRouterModel, component_definition=partition_router_manifest, config=input_config
406
- )
407
-
408
- assert isinstance(partition_router, ListPartitionRouter)
409
- assert partition_router.values == ["airbyte", "airbyte-cloud"]
410
-
411
-
412
- def test_list_based_stream_slicer_with_values_defined_in_config():
413
- content = """
414
- partition_router:
415
- type: ListPartitionRouter
416
- values: "{{config['repos']}}"
417
- cursor_field: repository
418
- request_option:
419
- type: RequestOption
420
- inject_into: header
421
- field_name: repository
422
- """
423
- parsed_manifest = YamlDeclarativeSource._parse(content)
424
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
425
- partition_router_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["partition_router"], {})
426
-
427
- partition_router = factory.create_component(
428
- model_type=ListPartitionRouterModel, component_definition=partition_router_manifest, config=input_config
429
- )
430
-
431
- assert isinstance(partition_router, ListPartitionRouter)
432
- assert partition_router.values == ["airbyte", "airbyte-cloud"]
433
- assert partition_router.request_option.inject_into == RequestOptionType.header
434
- assert partition_router.request_option.field_name.eval(config=input_config) == "repository"
435
-
436
-
437
- def test_create_substream_partition_router():
438
- content = """
439
- schema_loader:
440
- file_path: "./source_sendgrid/schemas/{{ parameters['name'] }}.yaml"
441
- name: "{{ parameters['stream_name'] }}"
442
- retriever:
443
- requester:
444
- type: "HttpRequester"
445
- path: "kek"
446
- record_selector:
447
- extractor:
448
- field_path: []
449
- stream_A:
450
- type: DeclarativeStream
451
- name: "A"
452
- primary_key: "id"
453
- $parameters:
454
- retriever: "#/retriever"
455
- url_base: "https://airbyte.io"
456
- schema_loader: "#/schema_loader"
457
- stream_B:
458
- type: DeclarativeStream
459
- name: "B"
460
- primary_key: "id"
461
- $parameters:
462
- retriever: "#/retriever"
463
- url_base: "https://airbyte.io"
464
- schema_loader: "#/schema_loader"
465
- partition_router:
466
- type: SubstreamPartitionRouter
467
- parent_stream_configs:
468
- - stream: "#/stream_A"
469
- parent_key: id
470
- partition_field: repository_id
471
- request_option:
472
- type: RequestOption
473
- inject_into: request_parameter
474
- field_name: repository_id
475
- - stream: "#/stream_B"
476
- parent_key: someid
477
- partition_field: word_id
478
- """
479
- parsed_manifest = YamlDeclarativeSource._parse(content)
480
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
481
- partition_router_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["partition_router"], {})
482
-
483
- partition_router = factory.create_component(
484
- model_type=SubstreamPartitionRouterModel, component_definition=partition_router_manifest, config=input_config
485
- )
486
-
487
- assert isinstance(partition_router, SubstreamPartitionRouter)
488
- parent_stream_configs = partition_router.parent_stream_configs
489
- assert len(parent_stream_configs) == 2
490
- assert isinstance(parent_stream_configs[0].stream, DeclarativeStream)
491
- assert isinstance(parent_stream_configs[1].stream, DeclarativeStream)
492
-
493
- assert partition_router.parent_stream_configs[0].parent_key.eval({}) == "id"
494
- assert partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id"
495
- assert partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter
496
- assert partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id"
497
-
498
- assert partition_router.parent_stream_configs[1].parent_key.eval({}) == "someid"
499
- assert partition_router.parent_stream_configs[1].partition_field.eval({}) == "word_id"
500
- assert partition_router.parent_stream_configs[1].request_option is None
501
-
502
-
503
- def test_datetime_based_cursor():
504
- content = """
505
- incremental:
506
- type: DatetimeBasedCursor
507
- $parameters:
508
- datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
509
- start_datetime:
510
- type: MinMaxDatetime
511
- datetime: "{{ config['start_time'] }}"
512
- min_datetime: "{{ config['start_time'] + day_delta(2) }}"
513
- end_datetime: "{{ config['end_time'] }}"
514
- step: "P10D"
515
- cursor_field: "created"
516
- cursor_granularity: "PT0.000001S"
517
- lookback_window: "P5D"
518
- start_time_option:
519
- type: RequestOption
520
- inject_into: request_parameter
521
- field_name: "since_{{ config['cursor_field'] }}"
522
- end_time_option:
523
- type: RequestOption
524
- inject_into: body_json
525
- field_name: "before_{{ parameters['cursor_field'] }}"
526
- partition_field_start: star
527
- partition_field_end: en
528
- """
529
- parsed_manifest = YamlDeclarativeSource._parse(content)
530
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
531
- slicer_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["incremental"], {"cursor_field": "created_at"})
532
-
533
- stream_slicer = factory.create_component(model_type=DatetimeBasedCursorModel, component_definition=slicer_manifest, config=input_config)
534
-
535
- assert isinstance(stream_slicer, DatetimeBasedCursor)
536
- assert stream_slicer._step == datetime.timedelta(days=10)
537
- assert stream_slicer._cursor_field.string == "created"
538
- assert stream_slicer.cursor_granularity == "PT0.000001S"
539
- assert stream_slicer._lookback_window.string == "P5D"
540
- assert stream_slicer.start_time_option.inject_into == RequestOptionType.request_parameter
541
- assert stream_slicer.start_time_option.field_name.eval(config=input_config | {"cursor_field": "updated_at"}) == "since_updated_at"
542
- assert stream_slicer.end_time_option.inject_into == RequestOptionType.body_json
543
- assert stream_slicer.end_time_option.field_name.eval({}) == "before_created_at"
544
- assert stream_slicer._partition_field_start.eval({}) == "star"
545
- assert stream_slicer._partition_field_end.eval({}) == "en"
546
-
547
- assert isinstance(stream_slicer._start_datetime, MinMaxDatetime)
548
- assert stream_slicer.start_datetime._datetime_format == "%Y-%m-%dT%H:%M:%S.%f%z"
549
- assert stream_slicer.start_datetime.datetime.string == "{{ config['start_time'] }}"
550
- assert stream_slicer.start_datetime.min_datetime.string == "{{ config['start_time'] + day_delta(2) }}"
551
-
552
- assert isinstance(stream_slicer._end_datetime, MinMaxDatetime)
553
- assert stream_slicer._end_datetime.datetime.string == "{{ config['end_time'] }}"
554
-
555
-
556
- def test_stream_with_incremental_and_retriever_with_partition_router():
557
- content = """
558
- decoder:
559
- type: JsonDecoder
560
- extractor:
561
- type: DpathExtractor
562
- decoder: "#/decoder"
563
- selector:
564
- type: RecordSelector
565
- record_filter:
566
- type: RecordFilter
567
- condition: "{{ record['id'] > stream_state['id'] }}"
568
- requester:
569
- type: HttpRequester
570
- name: "{{ parameters['name'] }}"
571
- url_base: "https://api.sendgrid.com/v3/"
572
- http_method: "GET"
573
- authenticator:
574
- type: BearerAuthenticator
575
- api_token: "{{ config['apikey'] }}"
576
- request_parameters:
577
- unit: "day"
578
- list_stream:
579
- type: DeclarativeStream
580
- schema_loader:
581
- type: JsonFileSchemaLoader
582
- file_path: "./source_sendgrid/schemas/{{ parameters.name }}.json"
583
- incremental_sync:
584
- type: DatetimeBasedCursor
585
- $parameters:
586
- datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
587
- start_datetime: "{{ config['start_time'] }}"
588
- end_datetime: "{{ config['end_time'] }}"
589
- step: "P10D"
590
- cursor_field: "created"
591
- cursor_granularity: "PT0.000001S"
592
- lookback_window: "P5D"
593
- start_time_option:
594
- inject_into: request_parameter
595
- field_name: created[gte]
596
- end_time_option:
597
- inject_into: body_json
598
- field_name: end_time
599
- partition_field_start: star
600
- partition_field_end: en
601
- retriever:
602
- type: SimpleRetriever
603
- name: "{{ parameters['name'] }}"
604
- partition_router:
605
- type: ListPartitionRouter
606
- values: "{{config['repos']}}"
607
- cursor_field: a_key
608
- request_option:
609
- inject_into: header
610
- field_name: a_key
611
- paginator:
612
- type: DefaultPaginator
613
- page_size_option:
614
- inject_into: request_parameter
615
- field_name: page_size
616
- page_token_option:
617
- inject_into: path
618
- type: RequestPath
619
- pagination_strategy:
620
- type: "CursorPagination"
621
- cursor_value: "{{ response._metadata.next }}"
622
- page_size: 10
623
- requester:
624
- $ref: "#/requester"
625
- path: "{{ next_page_token['next_page_url'] }}"
626
- record_selector:
627
- $ref: "#/selector"
628
- $parameters:
629
- name: "lists"
630
- primary_key: "id"
631
- extractor:
632
- $ref: "#/extractor"
633
- field_path: ["{{ parameters['name'] }}"]
634
- """
635
-
636
- parsed_manifest = YamlDeclarativeSource._parse(content)
637
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
638
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["list_stream"], {})
639
-
640
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
641
-
642
- assert isinstance(stream, DeclarativeStream)
643
- assert isinstance(stream.retriever, SimpleRetriever)
644
- assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor)
645
-
646
- datetime_stream_slicer = stream.retriever.stream_slicer._cursor_factory.create()
647
- assert isinstance(datetime_stream_slicer, DatetimeBasedCursor)
648
- assert isinstance(datetime_stream_slicer._start_datetime, MinMaxDatetime)
649
- assert datetime_stream_slicer._start_datetime.datetime.string == "{{ config['start_time'] }}"
650
- assert isinstance(datetime_stream_slicer._end_datetime, MinMaxDatetime)
651
- assert datetime_stream_slicer._end_datetime.datetime.string == "{{ config['end_time'] }}"
652
- assert datetime_stream_slicer.step == "P10D"
653
- assert datetime_stream_slicer._cursor_field.string == "created"
654
-
655
- list_stream_slicer = stream.retriever.stream_slicer._partition_router
656
- assert isinstance(list_stream_slicer, ListPartitionRouter)
657
- assert list_stream_slicer.values == ["airbyte", "airbyte-cloud"]
658
- assert list_stream_slicer._cursor_field.string == "a_key"
659
-
660
-
661
- def test_incremental_data_feed():
662
- content = """
663
- selector:
664
- type: RecordSelector
665
- extractor:
666
- type: DpathExtractor
667
- field_path: ["extractor_path"]
668
- record_filter:
669
- type: RecordFilter
670
- condition: "{{ record['id'] > stream_state['id'] }}"
671
- requester:
672
- type: HttpRequester
673
- name: "{{ parameters['name'] }}"
674
- url_base: "https://api.sendgrid.com/v3/"
675
- http_method: "GET"
676
- list_stream:
677
- type: DeclarativeStream
678
- incremental_sync:
679
- type: DatetimeBasedCursor
680
- $parameters:
681
- datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
682
- start_datetime: "{{ config['start_time'] }}"
683
- cursor_field: "created"
684
- is_data_feed: true
685
- retriever:
686
- type: SimpleRetriever
687
- name: "{{ parameters['name'] }}"
688
- paginator:
689
- type: DefaultPaginator
690
- pagination_strategy:
691
- type: "CursorPagination"
692
- cursor_value: "{{ response._metadata.next }}"
693
- page_size: 10
694
- requester:
695
- $ref: "#/requester"
696
- path: "/"
697
- record_selector:
698
- $ref: "#/selector"
699
- $parameters:
700
- name: "lists"
701
- """
702
-
703
- parsed_manifest = YamlDeclarativeSource._parse(content)
704
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
705
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["list_stream"], {})
706
-
707
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
708
-
709
- assert isinstance(stream.retriever.paginator.pagination_strategy, StopConditionPaginationStrategyDecorator)
710
-
711
-
712
- def test_given_data_feed_and_incremental_then_raise_error():
713
- content = """
714
- incremental_sync:
715
- type: DatetimeBasedCursor
716
- $parameters:
717
- datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
718
- start_datetime: "{{ config['start_time'] }}"
719
- end_datetime: "2023-01-01"
720
- cursor_field: "created"
721
- is_data_feed: true"""
722
-
723
- parsed_incremental_sync = YamlDeclarativeSource._parse(content)
724
- resolved_incremental_sync = resolver.preprocess_manifest(parsed_incremental_sync)
725
- datetime_based_cursor_definition = transformer.propagate_types_and_parameters("", resolved_incremental_sync["incremental_sync"], {})
726
-
727
- with pytest.raises(ValueError):
728
- factory.create_component(
729
- model_type=DatetimeBasedCursorModel, component_definition=datetime_based_cursor_definition, config=input_config
730
- )
731
-
732
-
733
- @pytest.mark.parametrize(
734
- "test_name, record_selector, expected_runtime_selector",
735
- [("test_static_record_selector", "result", "result"), ("test_options_record_selector", "{{ parameters['name'] }}", "lists")],
736
- )
737
- def test_create_record_selector(test_name, record_selector, expected_runtime_selector):
738
- content = f"""
739
- extractor:
740
- type: DpathExtractor
741
- selector:
742
- $parameters:
743
- name: "lists"
744
- type: RecordSelector
745
- record_filter:
746
- type: RecordFilter
747
- condition: "{{{{ record['id'] > stream_state['id'] }}}}"
748
- extractor:
749
- $ref: "#/extractor"
750
- field_path: ["{record_selector}"]
751
- """
752
- parsed_manifest = YamlDeclarativeSource._parse(content)
753
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
754
- selector_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["selector"], {})
755
-
756
- selector = factory.create_component(
757
- model_type=RecordSelectorModel, component_definition=selector_manifest, transformations=[], config=input_config
758
- )
759
-
760
- assert isinstance(selector, RecordSelector)
761
- assert isinstance(selector.extractor, DpathExtractor)
762
- assert [fp.eval(input_config) for fp in selector.extractor.field_path] == [expected_runtime_selector]
763
- assert isinstance(selector.record_filter, RecordFilter)
764
- assert selector.record_filter.condition == "{{ record['id'] > stream_state['id'] }}"
765
-
766
-
767
- @pytest.mark.parametrize(
768
- "test_name, error_handler, expected_backoff_strategy_type",
769
- [
770
- (
771
- "test_create_requester_constant_error_handler",
772
- """
773
- error_handler:
774
- backoff_strategies:
775
- - type: "ConstantBackoffStrategy"
776
- backoff_time_in_seconds: 5
777
- """,
778
- ConstantBackoffStrategy,
779
- ),
780
- (
781
- "test_create_requester_exponential_error_handler",
782
- """
783
- error_handler:
784
- backoff_strategies:
785
- - type: "ExponentialBackoffStrategy"
786
- factor: 5
787
- """,
788
- ExponentialBackoffStrategy,
789
- ),
790
- (
791
- "test_create_requester_wait_time_from_header_error_handler",
792
- """
793
- error_handler:
794
- backoff_strategies:
795
- - type: "WaitTimeFromHeader"
796
- header: "a_header"
797
- """,
798
- WaitTimeFromHeaderBackoffStrategy,
799
- ),
800
- (
801
- "test_create_requester_wait_time_until_from_header_error_handler",
802
- """
803
- error_handler:
804
- backoff_strategies:
805
- - type: "WaitUntilTimeFromHeader"
806
- header: "a_header"
807
- """,
808
- WaitUntilTimeFromHeaderBackoffStrategy,
809
- ),
810
- ("test_create_requester_no_error_handler", """""", ExponentialBackoffStrategy),
811
- ],
812
- )
813
- def test_create_requester(test_name, error_handler, expected_backoff_strategy_type):
814
- content = f"""
815
- requester:
816
- type: HttpRequester
817
- path: "/v3/marketing/lists"
818
- $parameters:
819
- name: 'lists'
820
- url_base: "https://api.sendgrid.com"
821
- authenticator:
822
- type: "BasicHttpAuthenticator"
823
- username: "{{{{ parameters.name}}}}"
824
- password: "{{{{ config.apikey }}}}"
825
- request_parameters:
826
- a_parameter: "something_here"
827
- request_headers:
828
- header: header_value
829
- {error_handler}
830
- """
831
- name = "name"
832
- parsed_manifest = YamlDeclarativeSource._parse(content)
833
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
834
- requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {})
835
-
836
- selector = factory.create_component(
837
- model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name
838
- )
839
-
840
- assert isinstance(selector, HttpRequester)
841
- assert selector.http_method == HttpMethod.GET
842
- assert selector.name == "name"
843
- assert selector._path.string == "/v3/marketing/lists"
844
- assert selector._url_base.string == "https://api.sendgrid.com"
845
-
846
- assert isinstance(selector.error_handler, DefaultErrorHandler)
847
- assert len(selector.error_handler.backoff_strategies) == 1
848
- assert isinstance(selector.error_handler.backoff_strategies[0], expected_backoff_strategy_type)
849
-
850
- assert isinstance(selector.authenticator, BasicHttpAuthenticator)
851
- assert selector.authenticator._username.eval(input_config) == "lists"
852
- assert selector.authenticator._password.eval(input_config) == "verysecrettoken"
853
-
854
- assert isinstance(selector._request_options_provider, InterpolatedRequestOptionsProvider)
855
- assert selector._request_options_provider._parameter_interpolator._interpolator.mapping["a_parameter"] == "something_here"
856
- assert selector._request_options_provider._headers_interpolator._interpolator.mapping["header"] == "header_value"
857
-
858
-
859
- def test_create_request_with_leacy_session_authenticator():
860
- content = """
861
- requester:
862
- type: HttpRequester
863
- path: "/v3/marketing/lists"
864
- $parameters:
865
- name: 'lists'
866
- url_base: "https://api.sendgrid.com"
867
- authenticator:
868
- type: "LegacySessionTokenAuthenticator"
869
- username: "{{ parameters.name}}"
870
- password: "{{ config.apikey }}"
871
- login_url: "login"
872
- header: "token"
873
- session_token_response_key: "session"
874
- validate_session_url: validate
875
- request_parameters:
876
- a_parameter: "something_here"
877
- request_headers:
878
- header: header_value
879
- """
880
- name = "name"
881
- parsed_manifest = YamlDeclarativeSource._parse(content)
882
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
883
- requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {})
884
-
885
- selector = factory.create_component(
886
- model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name
887
- )
888
-
889
- assert isinstance(selector, HttpRequester)
890
- assert isinstance(selector.authenticator, LegacySessionTokenAuthenticator)
891
- assert selector.authenticator._username.eval(input_config) == "lists"
892
- assert selector.authenticator._password.eval(input_config) == "verysecrettoken"
893
- assert selector.authenticator._api_url.eval(input_config) == "https://api.sendgrid.com"
894
-
895
-
896
- def test_create_request_with_session_authenticator():
897
- content = """
898
- requester:
899
- type: HttpRequester
900
- path: "/v3/marketing/lists"
901
- $parameters:
902
- name: 'lists'
903
- url_base: "https://api.sendgrid.com"
904
- authenticator:
905
- type: SessionTokenAuthenticator
906
- expiration_duration: P10D
907
- login_requester:
908
- path: /session
909
- type: HttpRequester
910
- url_base: 'https://api.sendgrid.com'
911
- http_method: POST
912
- request_body_json:
913
- password: '{{ config.apikey }}'
914
- username: '{{ parameters.name }}'
915
- session_token_path:
916
- - id
917
- request_authentication:
918
- type: ApiKey
919
- inject_into:
920
- type: RequestOption
921
- field_name: X-Metabase-Session
922
- inject_into: header
923
- request_parameters:
924
- a_parameter: "something_here"
925
- request_headers:
926
- header: header_value
927
- """
928
- name = "name"
929
- parsed_manifest = YamlDeclarativeSource._parse(content)
930
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
931
- requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {})
932
-
933
- selector = factory.create_component(
934
- model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name
935
- )
936
-
937
- assert isinstance(selector.authenticator, ApiKeyAuthenticator)
938
- assert isinstance(selector.authenticator.token_provider, SessionTokenProvider)
939
- assert selector.authenticator.token_provider.session_token_path == ["id"]
940
- assert isinstance(selector.authenticator.token_provider.login_requester, HttpRequester)
941
- assert selector.authenticator.token_provider.session_token_path == ["id"]
942
- assert selector.authenticator.token_provider.login_requester._url_base.eval(input_config) == "https://api.sendgrid.com"
943
- assert selector.authenticator.token_provider.login_requester.get_request_body_json() == {
944
- "username": "lists",
945
- "password": "verysecrettoken",
946
- }
947
-
948
-
949
- @pytest.mark.parametrize(
950
- "input_config, expected_authenticator_class",
951
- [
952
- pytest.param(
953
- {"auth": {"type": "token"}, "credentials": {"api_key": "some_key"}},
954
- ApiKeyAuthenticator,
955
- id="test_create_requester_with_selective_authenticator_and_token_selected",
956
- ),
957
- pytest.param(
958
- {"auth": {"type": "oauth"}, "credentials": {"client_id": "ABC"}},
959
- DeclarativeOauth2Authenticator,
960
- id="test_create_requester_with_selective_authenticator_and_oauth_selected",
961
- ),
962
- ],
963
- )
964
- def test_create_requester_with_selective_authenticator(input_config, expected_authenticator_class):
965
- content = """
966
- authenticator:
967
- type: SelectiveAuthenticator
968
- authenticator_selection_path:
969
- - auth
970
- - type
971
- authenticators:
972
- token:
973
- type: ApiKeyAuthenticator
974
- header: "Authorization"
975
- api_token: "api_key={{ config['credentials']['api_key'] }}"
976
- oauth:
977
- type: OAuthAuthenticator
978
- token_refresh_endpoint: https://api.url.com
979
- client_id: "{{ config['credentials']['client_id'] }}"
980
- client_secret: some_secret
981
- refresh_token: some_token
982
- """
983
- name = "name"
984
- parsed_manifest = YamlDeclarativeSource._parse(content)
985
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
986
- authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
987
-
988
- authenticator = factory.create_component(
989
- model_type=SelectiveAuthenticator, component_definition=authenticator_manifest, config=input_config, name=name
990
- )
991
-
992
- assert isinstance(authenticator, expected_authenticator_class)
993
-
994
-
995
- def test_create_composite_error_handler():
996
- content = """
997
- error_handler:
998
- type: "CompositeErrorHandler"
999
- error_handlers:
1000
- - response_filters:
1001
- - predicate: "{{ 'code' in response }}"
1002
- action: RETRY
1003
- - response_filters:
1004
- - http_codes: [ 403 ]
1005
- action: RETRY
1006
- """
1007
- parsed_manifest = YamlDeclarativeSource._parse(content)
1008
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1009
- error_handler_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["error_handler"], {})
1010
-
1011
- error_handler = factory.create_component(
1012
- model_type=CompositeErrorHandlerModel, component_definition=error_handler_manifest, config=input_config
1013
- )
1014
-
1015
- assert isinstance(error_handler, CompositeErrorHandler)
1016
- assert len(error_handler.error_handlers) == 2
1017
-
1018
- error_handler_0 = error_handler.error_handlers[0]
1019
- assert isinstance(error_handler_0, DefaultErrorHandler)
1020
- assert isinstance(error_handler_0.response_filters[0], HttpResponseFilter)
1021
- assert error_handler_0.response_filters[0].predicate.condition == "{{ 'code' in response }}"
1022
- assert error_handler_0.response_filters[0].action == ResponseAction.RETRY
1023
-
1024
- error_handler_1 = error_handler.error_handlers[1]
1025
- assert isinstance(error_handler_1, DefaultErrorHandler)
1026
- assert isinstance(error_handler_1.response_filters[0], HttpResponseFilter)
1027
- assert error_handler_1.response_filters[0].http_codes == {403}
1028
- assert error_handler_1.response_filters[0].action == ResponseAction.RETRY
1029
-
1030
-
1031
- # This might be a better test for the manifest transformer but also worth testing end-to-end here as well
1032
- def test_config_with_defaults():
1033
- content = """
1034
- lists_stream:
1035
- type: "DeclarativeStream"
1036
- name: "lists"
1037
- primary_key: id
1038
- $parameters:
1039
- name: "lists"
1040
- url_base: "https://api.sendgrid.com"
1041
- schema_loader:
1042
- name: "{{ parameters.stream_name }}"
1043
- file_path: "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
1044
- retriever:
1045
- paginator:
1046
- type: "DefaultPaginator"
1047
- page_size_option:
1048
- type: RequestOption
1049
- inject_into: request_parameter
1050
- field_name: page_size
1051
- page_token_option:
1052
- type: RequestPath
1053
- pagination_strategy:
1054
- type: "CursorPagination"
1055
- cursor_value: "{{ response._metadata.next }}"
1056
- page_size: 10
1057
- requester:
1058
- path: "/v3/marketing/lists"
1059
- authenticator:
1060
- type: "BearerAuthenticator"
1061
- api_token: "{{ config.apikey }}"
1062
- request_parameters:
1063
- page_size: 10
1064
- record_selector:
1065
- extractor:
1066
- field_path: ["result"]
1067
- streams:
1068
- - "#/lists_stream"
1069
- """
1070
- parsed_manifest = YamlDeclarativeSource._parse(content)
1071
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1072
- resolved_manifest["type"] = "DeclarativeSource"
1073
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["lists_stream"], {})
1074
-
1075
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
1076
-
1077
- assert isinstance(stream, DeclarativeStream)
1078
- assert stream.primary_key == "id"
1079
- assert stream.name == "lists"
1080
- assert isinstance(stream.retriever, SimpleRetriever)
1081
- assert stream.retriever.name == stream.name
1082
- assert stream.retriever.primary_key == stream.primary_key
1083
-
1084
- assert isinstance(stream.schema_loader, JsonFileSchemaLoader)
1085
- assert stream.schema_loader.file_path.string == "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
1086
- assert stream.schema_loader.file_path.default == "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
1087
-
1088
- assert isinstance(stream.retriever.requester, HttpRequester)
1089
- assert stream.retriever.requester.http_method == HttpMethod.GET
1090
-
1091
- assert isinstance(stream.retriever.requester.authenticator, BearerAuthenticator)
1092
- assert stream.retriever.requester.authenticator.token_provider.get_token() == "verysecrettoken"
1093
-
1094
- assert isinstance(stream.retriever.record_selector, RecordSelector)
1095
- assert isinstance(stream.retriever.record_selector.extractor, DpathExtractor)
1096
- assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_path] == ["result"]
1097
-
1098
- assert isinstance(stream.retriever.paginator, DefaultPaginator)
1099
- assert stream.retriever.paginator.url_base.string == "https://api.sendgrid.com"
1100
- assert stream.retriever.paginator.pagination_strategy.get_page_size() == 10
1101
-
1102
-
1103
- def test_create_default_paginator():
1104
- content = """
1105
- paginator:
1106
- type: "DefaultPaginator"
1107
- page_size_option:
1108
- type: RequestOption
1109
- inject_into: request_parameter
1110
- field_name: page_size
1111
- page_token_option:
1112
- type: RequestPath
1113
- pagination_strategy:
1114
- type: "CursorPagination"
1115
- page_size: 50
1116
- cursor_value: "{{ response._metadata.next }}"
1117
- """
1118
- parsed_manifest = YamlDeclarativeSource._parse(content)
1119
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1120
- paginator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["paginator"], {})
1121
-
1122
- paginator = factory.create_component(
1123
- model_type=DefaultPaginatorModel, component_definition=paginator_manifest, config=input_config, url_base="https://airbyte.io"
1124
- )
1125
-
1126
- assert isinstance(paginator, DefaultPaginator)
1127
- assert paginator.url_base.string == "https://airbyte.io"
1128
-
1129
- assert isinstance(paginator.pagination_strategy, CursorPaginationStrategy)
1130
- assert paginator.pagination_strategy.page_size == 50
1131
- assert paginator.pagination_strategy.cursor_value.string == "{{ response._metadata.next }}"
1132
-
1133
- assert isinstance(paginator.page_size_option, RequestOption)
1134
- assert paginator.page_size_option.inject_into == RequestOptionType.request_parameter
1135
- assert paginator.page_size_option.field_name.eval(config=input_config) == "page_size"
1136
-
1137
- assert isinstance(paginator.page_token_option, RequestPath)
1138
-
1139
-
1140
- @pytest.mark.parametrize(
1141
- "manifest, field_name, expected_value, expected_error",
1142
- [
1143
- pytest.param(
1144
- {
1145
- "type": "CustomErrorHandler",
1146
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1147
- "subcomponent_field_with_hint": {"type": "DpathExtractor", "field_path": []},
1148
- },
1149
- "subcomponent_field_with_hint",
1150
- DpathExtractor(field_path=[], config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, parameters={}),
1151
- None,
1152
- id="test_create_custom_component_with_subcomponent_that_must_be_parsed",
1153
- ),
1154
- pytest.param(
1155
- {
1156
- "type": "CustomErrorHandler",
1157
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1158
- "subcomponent_field_with_hint": {"field_path": []},
1159
- },
1160
- "subcomponent_field_with_hint",
1161
- DpathExtractor(field_path=[], config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, parameters={}),
1162
- None,
1163
- id="test_create_custom_component_with_subcomponent_that_must_infer_type_from_explicit_hints",
1164
- ),
1165
- pytest.param(
1166
- {
1167
- "type": "CustomErrorHandler",
1168
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1169
- "basic_field": "expected",
1170
- },
1171
- "basic_field",
1172
- "expected",
1173
- None,
1174
- id="test_create_custom_component_with_built_in_type",
1175
- ),
1176
- pytest.param(
1177
- {
1178
- "type": "CustomErrorHandler",
1179
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1180
- "optional_subcomponent_field": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "destination"},
1181
- },
1182
- "optional_subcomponent_field",
1183
- RequestOption(inject_into=RequestOptionType.request_parameter, field_name="destination", parameters={}),
1184
- None,
1185
- id="test_create_custom_component_with_subcomponent_wrapped_in_optional",
1186
- ),
1187
- pytest.param(
1188
- {
1189
- "type": "CustomErrorHandler",
1190
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1191
- "list_of_subcomponents": [
1192
- {"inject_into": "header", "field_name": "store_me"},
1193
- {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "destination"},
1194
- ],
1195
- },
1196
- "list_of_subcomponents",
1197
- [
1198
- RequestOption(inject_into=RequestOptionType.header, field_name="store_me", parameters={}),
1199
- RequestOption(inject_into=RequestOptionType.request_parameter, field_name="destination", parameters={}),
1200
- ],
1201
- None,
1202
- id="test_create_custom_component_with_subcomponent_wrapped_in_list",
1203
- ),
1204
- pytest.param(
1205
- {
1206
- "type": "CustomErrorHandler",
1207
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1208
- "without_hint": {"inject_into": "request_parameter", "field_name": "missing_hint"},
1209
- },
1210
- "without_hint",
1211
- None,
1212
- None,
1213
- id="test_create_custom_component_with_subcomponent_without_type_hints",
1214
- ),
1215
- pytest.param(
1216
- {
1217
- "type": "CustomErrorHandler",
1218
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1219
- "paginator": {
1220
- "type": "DefaultPaginator",
1221
- "pagination_strategy": {"type": "OffsetIncrement", "page_size": 10},
1222
- "$parameters": {"url_base": "https://physical_100.com"},
1223
- },
1224
- },
1225
- "paginator",
1226
- DefaultPaginator(
1227
- pagination_strategy=OffsetIncrement(
1228
- page_size=10, config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, parameters={}
1229
- ),
1230
- url_base="https://physical_100.com",
1231
- config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]},
1232
- parameters={},
1233
- ),
1234
- None,
1235
- id="test_create_custom_component_with_subcomponent_that_uses_parameters",
1236
- ),
1237
- pytest.param(
1238
- {
1239
- "type": "CustomErrorHandler",
1240
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
1241
- "paginator": {
1242
- "type": "DefaultPaginator",
1243
- "pagination_strategy": {"type": "OffsetIncrement", "page_size": 10},
1244
- },
1245
- },
1246
- "paginator",
1247
- None,
1248
- ValueError,
1249
- id="test_create_custom_component_missing_required_field_emits_error",
1250
- ),
1251
- pytest.param(
1252
- {
1253
- "type": "CustomErrorHandler",
1254
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.NonExistingClass",
1255
- "paginator": {
1256
- "type": "DefaultPaginator",
1257
- "pagination_strategy": {"type": "OffsetIncrement", "page_size": 10},
1258
- },
1259
- },
1260
- "paginator",
1261
- None,
1262
- ValueError,
1263
- id="test_create_custom_component_non_existing_class_raises_value_error",
1264
- ),
1265
- ],
1266
- )
1267
- def test_create_custom_components(manifest, field_name, expected_value, expected_error):
1268
- if expected_error:
1269
- with pytest.raises(expected_error):
1270
- factory.create_component(CustomErrorHandlerModel, manifest, input_config)
1271
- else:
1272
- custom_component = factory.create_component(CustomErrorHandlerModel, manifest, input_config)
1273
- assert isinstance(custom_component, TestingSomeComponent)
1274
-
1275
- assert isinstance(getattr(custom_component, field_name), type(expected_value))
1276
- assert getattr(custom_component, field_name) == expected_value
1277
-
1278
-
1279
- def test_custom_components_do_not_contain_extra_fields():
1280
- custom_substream_partition_router_manifest = {
1281
- "type": "CustomPartitionRouter",
1282
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingCustomSubstreamPartitionRouter",
1283
- "custom_field": "here",
1284
- "extra_field_to_exclude": "should_not_pass_as_parameter",
1285
- "custom_pagination_strategy": {"type": "PageIncrement", "page_size": 100},
1286
- "parent_stream_configs": [
1287
- {
1288
- "type": "ParentStreamConfig",
1289
- "stream": {
1290
- "type": "DeclarativeStream",
1291
- "name": "a_parent",
1292
- "primary_key": "id",
1293
- "retriever": {
1294
- "type": "SimpleRetriever",
1295
- "record_selector": {
1296
- "type": "RecordSelector",
1297
- "extractor": {"type": "DpathExtractor", "field_path": []},
1298
- },
1299
- "requester": {"type": "HttpRequester", "url_base": "https://airbyte.io", "path": "some"},
1300
- },
1301
- "schema_loader": {
1302
- "type": "JsonFileSchemaLoader",
1303
- "file_path": "./source_sendgrid/schemas/{{ parameters['name'] }}.yaml",
1304
- },
1305
- },
1306
- "parent_key": "id",
1307
- "partition_field": "repository_id",
1308
- "request_option": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "repository_id"},
1309
- }
1310
- ],
1311
- }
1312
-
1313
- custom_substream_partition_router = factory.create_component(
1314
- CustomPartitionRouterModel, custom_substream_partition_router_manifest, input_config
1315
- )
1316
- assert isinstance(custom_substream_partition_router, TestingCustomSubstreamPartitionRouter)
1317
-
1318
- assert len(custom_substream_partition_router.parent_stream_configs) == 1
1319
- assert custom_substream_partition_router.parent_stream_configs[0].parent_key.eval({}) == "id"
1320
- assert custom_substream_partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id"
1321
- assert custom_substream_partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter
1322
- assert custom_substream_partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id"
1323
-
1324
- assert isinstance(custom_substream_partition_router.custom_pagination_strategy, PageIncrement)
1325
- assert custom_substream_partition_router.custom_pagination_strategy.page_size == 100
1326
-
1327
-
1328
- def test_parse_custom_component_fields_if_subcomponent():
1329
- custom_substream_partition_router_manifest = {
1330
- "type": "CustomPartitionRouter",
1331
- "class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingCustomSubstreamPartitionRouter",
1332
- "custom_field": "here",
1333
- "custom_pagination_strategy": {"type": "PageIncrement", "page_size": 100},
1334
- "parent_stream_configs": [
1335
- {
1336
- "type": "ParentStreamConfig",
1337
- "stream": {
1338
- "type": "DeclarativeStream",
1339
- "name": "a_parent",
1340
- "primary_key": "id",
1341
- "retriever": {
1342
- "type": "SimpleRetriever",
1343
- "record_selector": {
1344
- "type": "RecordSelector",
1345
- "extractor": {"type": "DpathExtractor", "field_path": []},
1346
- },
1347
- "requester": {"type": "HttpRequester", "url_base": "https://airbyte.io", "path": "some"},
1348
- },
1349
- "schema_loader": {
1350
- "type": "JsonFileSchemaLoader",
1351
- "file_path": "./source_sendgrid/schemas/{{ parameters['name'] }}.yaml",
1352
- },
1353
- },
1354
- "parent_key": "id",
1355
- "partition_field": "repository_id",
1356
- "request_option": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "repository_id"},
1357
- }
1358
- ],
1359
- }
1360
-
1361
- custom_substream_partition_router = factory.create_component(
1362
- CustomPartitionRouterModel, custom_substream_partition_router_manifest, input_config
1363
- )
1364
- assert isinstance(custom_substream_partition_router, TestingCustomSubstreamPartitionRouter)
1365
- assert custom_substream_partition_router.custom_field == "here"
1366
-
1367
- assert len(custom_substream_partition_router.parent_stream_configs) == 1
1368
- assert custom_substream_partition_router.parent_stream_configs[0].parent_key.eval({}) == "id"
1369
- assert custom_substream_partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id"
1370
- assert custom_substream_partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter
1371
- assert custom_substream_partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id"
1372
-
1373
- assert isinstance(custom_substream_partition_router.custom_pagination_strategy, PageIncrement)
1374
- assert custom_substream_partition_router.custom_pagination_strategy.page_size == 100
1375
-
1376
-
1377
- class TestCreateTransformations:
1378
- # the tabbing matters
1379
- base_parameters = """
1380
- name: "lists"
1381
- primary_key: id
1382
- url_base: "https://api.sendgrid.com"
1383
- schema_loader:
1384
- name: "{{ parameters.name }}"
1385
- file_path: "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
1386
- retriever:
1387
- requester:
1388
- name: "{{ parameters.name }}"
1389
- path: "/v3/marketing/lists"
1390
- request_parameters:
1391
- page_size: 10
1392
- record_selector:
1393
- extractor:
1394
- field_path: ["result"]
1395
- """
1396
-
1397
- def test_no_transformations(self):
1398
- content = f"""
1399
- the_stream:
1400
- type: DeclarativeStream
1401
- $parameters:
1402
- {self.base_parameters}
1403
- """
1404
- parsed_manifest = YamlDeclarativeSource._parse(content)
1405
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1406
- resolved_manifest["type"] = "DeclarativeSource"
1407
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["the_stream"], {})
1408
-
1409
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
1410
-
1411
- assert isinstance(stream, DeclarativeStream)
1412
- assert [] == stream.retriever.record_selector.transformations
1413
-
1414
- def test_remove_fields(self):
1415
- content = f"""
1416
- the_stream:
1417
- type: DeclarativeStream
1418
- $parameters:
1419
- {self.base_parameters}
1420
- transformations:
1421
- - type: RemoveFields
1422
- field_pointers:
1423
- - ["path", "to", "field1"]
1424
- - ["path2"]
1425
- """
1426
- parsed_manifest = YamlDeclarativeSource._parse(content)
1427
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1428
- resolved_manifest["type"] = "DeclarativeSource"
1429
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["the_stream"], {})
1430
-
1431
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
1432
-
1433
- assert isinstance(stream, DeclarativeStream)
1434
- expected = [RemoveFields(field_pointers=[["path", "to", "field1"], ["path2"]], parameters={})]
1435
- assert stream.retriever.record_selector.transformations == expected
1436
-
1437
- def test_add_fields_no_value_type(self):
1438
- content = f"""
1439
- the_stream:
1440
- type: DeclarativeStream
1441
- $parameters:
1442
- {self.base_parameters}
1443
- transformations:
1444
- - type: AddFields
1445
- fields:
1446
- - path: ["field1"]
1447
- value: "static_value"
1448
- """
1449
- expected = [
1450
- AddFields(
1451
- fields=[
1452
- AddedFieldDefinition(
1453
- path=["field1"],
1454
- value=InterpolatedString(string="static_value", default="static_value", parameters={}),
1455
- value_type=None,
1456
- parameters={},
1457
- )
1458
- ],
1459
- parameters={},
1460
- )
1461
- ]
1462
- self._test_add_fields(content, expected)
1463
-
1464
- def test_add_fields_value_type_is_string(self):
1465
- content = f"""
1466
- the_stream:
1467
- type: DeclarativeStream
1468
- $parameters:
1469
- {self.base_parameters}
1470
- transformations:
1471
- - type: AddFields
1472
- fields:
1473
- - path: ["field1"]
1474
- value: "static_value"
1475
- value_type: string
1476
- """
1477
- expected = [
1478
- AddFields(
1479
- fields=[
1480
- AddedFieldDefinition(
1481
- path=["field1"],
1482
- value=InterpolatedString(string="static_value", default="static_value", parameters={}),
1483
- value_type=str,
1484
- parameters={},
1485
- )
1486
- ],
1487
- parameters={},
1488
- )
1489
- ]
1490
- self._test_add_fields(content, expected)
1491
-
1492
- def test_add_fields_value_type_is_number(self):
1493
- content = f"""
1494
- the_stream:
1495
- type: DeclarativeStream
1496
- $parameters:
1497
- {self.base_parameters}
1498
- transformations:
1499
- - type: AddFields
1500
- fields:
1501
- - path: ["field1"]
1502
- value: "1"
1503
- value_type: number
1504
- """
1505
- expected = [
1506
- AddFields(
1507
- fields=[
1508
- AddedFieldDefinition(
1509
- path=["field1"],
1510
- value=InterpolatedString(string="1", default="1", parameters={}),
1511
- value_type=float,
1512
- parameters={},
1513
- )
1514
- ],
1515
- parameters={},
1516
- )
1517
- ]
1518
- self._test_add_fields(content, expected)
1519
-
1520
- def test_add_fields_value_type_is_integer(self):
1521
- content = f"""
1522
- the_stream:
1523
- type: DeclarativeStream
1524
- $parameters:
1525
- {self.base_parameters}
1526
- transformations:
1527
- - type: AddFields
1528
- fields:
1529
- - path: ["field1"]
1530
- value: "1"
1531
- value_type: integer
1532
- """
1533
- expected = [
1534
- AddFields(
1535
- fields=[
1536
- AddedFieldDefinition(
1537
- path=["field1"],
1538
- value=InterpolatedString(string="1", default="1", parameters={}),
1539
- value_type=int,
1540
- parameters={},
1541
- )
1542
- ],
1543
- parameters={},
1544
- )
1545
- ]
1546
- self._test_add_fields(content, expected)
1547
-
1548
- def test_add_fields_value_type_is_boolean(self):
1549
- content = f"""
1550
- the_stream:
1551
- type: DeclarativeStream
1552
- $parameters:
1553
- {self.base_parameters}
1554
- transformations:
1555
- - type: AddFields
1556
- fields:
1557
- - path: ["field1"]
1558
- value: False
1559
- value_type: boolean
1560
- """
1561
- expected = [
1562
- AddFields(
1563
- fields=[
1564
- AddedFieldDefinition(
1565
- path=["field1"],
1566
- value=InterpolatedString(string="False", default="False", parameters={}),
1567
- value_type=bool,
1568
- parameters={},
1569
- )
1570
- ],
1571
- parameters={},
1572
- )
1573
- ]
1574
- self._test_add_fields(content, expected)
1575
-
1576
- def _test_add_fields(self, content, expected):
1577
- parsed_manifest = YamlDeclarativeSource._parse(content)
1578
- resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
1579
- resolved_manifest["type"] = "DeclarativeSource"
1580
- stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["the_stream"], {})
1581
-
1582
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
1583
-
1584
- assert isinstance(stream, DeclarativeStream)
1585
- assert stream.retriever.record_selector.transformations == expected
1586
-
1587
- def test_default_schema_loader(self):
1588
- component_definition = {
1589
- "type": "DeclarativeStream",
1590
- "name": "test",
1591
- "primary_key": [],
1592
- "retriever": {
1593
- "type": "SimpleRetriever",
1594
- "requester": {
1595
- "type": "HttpRequester",
1596
- "url_base": "http://localhost:6767/",
1597
- "path": "items/",
1598
- "request_options_provider": {
1599
- "request_parameters": {},
1600
- "request_headers": {},
1601
- "request_body_json": {},
1602
- "type": "InterpolatedRequestOptionsProvider",
1603
- },
1604
- "authenticator": {"type": "BearerAuthenticator", "api_token": "{{ config['api_key'] }}"},
1605
- },
1606
- "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": ["items"]}},
1607
- "paginator": {"type": "NoPagination"},
1608
- },
1609
- }
1610
- resolved_manifest = resolver.preprocess_manifest(component_definition)
1611
- ws = ManifestComponentTransformer()
1612
- propagated_source_config = ws.propagate_types_and_parameters("", resolved_manifest, {})
1613
- stream = factory.create_component(
1614
- model_type=DeclarativeStreamModel, component_definition=propagated_source_config, config=input_config
1615
- )
1616
- schema_loader = stream.schema_loader
1617
- assert schema_loader.default_loader._get_json_filepath().split("/")[-1] == f"{stream.name}.json"
1618
-
1619
-
1620
- @pytest.mark.parametrize(
1621
- "incremental, partition_router, expected_type",
1622
- [
1623
- pytest.param(
1624
- {
1625
- "type": "DatetimeBasedCursor",
1626
- "datetime_format": "%Y-%m-%dT%H:%M:%S.%f%z",
1627
- "start_datetime": "{{ config['start_time'] }}",
1628
- "end_datetime": "{{ config['end_time'] }}",
1629
- "step": "P10D",
1630
- "cursor_field": "created",
1631
- "cursor_granularity": "PT0.000001S",
1632
- },
1633
- None,
1634
- DatetimeBasedCursor,
1635
- id="test_create_simple_retriever_with_incremental",
1636
- ),
1637
- pytest.param(
1638
- None,
1639
- {
1640
- "type": "ListPartitionRouter",
1641
- "values": "{{config['repos']}}",
1642
- "cursor_field": "a_key",
1643
- },
1644
- ListPartitionRouter,
1645
- id="test_create_simple_retriever_with_partition_router",
1646
- ),
1647
- pytest.param(
1648
- {
1649
- "type": "DatetimeBasedCursor",
1650
- "datetime_format": "%Y-%m-%dT%H:%M:%S.%f%z",
1651
- "start_datetime": "{{ config['start_time'] }}",
1652
- "end_datetime": "{{ config['end_time'] }}",
1653
- "step": "P10D",
1654
- "cursor_field": "created",
1655
- "cursor_granularity": "PT0.000001S",
1656
- },
1657
- {
1658
- "type": "ListPartitionRouter",
1659
- "values": "{{config['repos']}}",
1660
- "cursor_field": "a_key",
1661
- },
1662
- PerPartitionCursor,
1663
- id="test_create_simple_retriever_with_incremental_and_partition_router",
1664
- ),
1665
- pytest.param(
1666
- {
1667
- "type": "DatetimeBasedCursor",
1668
- "datetime_format": "%Y-%m-%dT%H:%M:%S.%f%z",
1669
- "start_datetime": "{{ config['start_time'] }}",
1670
- "end_datetime": "{{ config['end_time'] }}",
1671
- "step": "P10D",
1672
- "cursor_field": "created",
1673
- "cursor_granularity": "PT0.000001S",
1674
- },
1675
- [
1676
- {
1677
- "type": "ListPartitionRouter",
1678
- "values": "{{config['repos']}}",
1679
- "cursor_field": "a_key",
1680
- },
1681
- {
1682
- "type": "ListPartitionRouter",
1683
- "values": "{{config['repos']}}",
1684
- "cursor_field": "b_key",
1685
- },
1686
- ],
1687
- PerPartitionCursor,
1688
- id="test_create_simple_retriever_with_partition_routers_multiple_components",
1689
- ),
1690
- pytest.param(None, None, SinglePartitionRouter, id="test_create_simple_retriever_with_no_incremental_or_partition_router"),
1691
- ],
1692
- )
1693
- def test_merge_incremental_and_partition_router(incremental, partition_router, expected_type):
1694
- stream_model = {
1695
- "type": "DeclarativeStream",
1696
- "retriever": {
1697
- "type": "SimpleRetriever",
1698
- "record_selector": {
1699
- "type": "RecordSelector",
1700
- "extractor": {
1701
- "type": "DpathExtractor",
1702
- "field_path": [],
1703
- },
1704
- },
1705
- "requester": {
1706
- "type": "HttpRequester",
1707
- "name": "list",
1708
- "url_base": "orange.com",
1709
- "path": "/v1/api",
1710
- },
1711
- },
1712
- }
1713
-
1714
- if incremental:
1715
- stream_model["incremental_sync"] = incremental
1716
-
1717
- if partition_router:
1718
- stream_model["retriever"]["partition_router"] = partition_router
1719
-
1720
- stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_model, config=input_config)
1721
-
1722
- assert isinstance(stream, DeclarativeStream)
1723
- assert isinstance(stream.retriever, SimpleRetriever)
1724
- assert isinstance(stream.retriever.stream_slicer, expected_type)
1725
-
1726
- if incremental and partition_router:
1727
- assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor)
1728
- if type(partition_router) == list and len(partition_router) > 1:
1729
- assert type(stream.retriever.stream_slicer._partition_router) == CartesianProductStreamSlicer
1730
- assert len(stream.retriever.stream_slicer._partition_router.stream_slicers) == len(partition_router)
1731
- elif partition_router and type(partition_router) == list and len(partition_router) > 1:
1732
- assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor)
1733
- assert len(stream.retriever.stream_slicer.stream_slicerS) == len(partition_router)
1734
-
1735
-
1736
- def test_simple_retriever_emit_log_messages():
1737
- simple_retriever_model = {
1738
- "type": "SimpleRetriever",
1739
- "record_selector": {
1740
- "type": "RecordSelector",
1741
- "extractor": {
1742
- "type": "DpathExtractor",
1743
- "field_path": [],
1744
- },
1745
- },
1746
- "requester": {"type": "HttpRequester", "name": "list", "url_base": "orange.com", "path": "/v1/api"},
1747
- }
1748
-
1749
- connector_builder_factory = ModelToComponentFactory(emit_connector_builder_messages=True)
1750
- retriever = connector_builder_factory.create_component(
1751
- model_type=SimpleRetrieverModel,
1752
- component_definition=simple_retriever_model,
1753
- config={},
1754
- name="Test",
1755
- primary_key="id",
1756
- stream_slicer=None,
1757
- transformations=[],
1758
- )
1759
-
1760
- assert isinstance(retriever, SimpleRetrieverTestReadDecorator)
1761
- assert connector_builder_factory._message_repository._log_level == Level.DEBUG
1762
-
1763
-
1764
- def test_ignore_retry():
1765
- requester_model = {
1766
- "type": "HttpRequester",
1767
- "name": "list",
1768
- "url_base": "orange.com",
1769
- "path": "/v1/api",
1770
- }
1771
-
1772
- connector_builder_factory = ModelToComponentFactory(disable_retries=True)
1773
- requester = connector_builder_factory.create_component(
1774
- model_type=HttpRequesterModel,
1775
- component_definition=requester_model,
1776
- config={},
1777
- name="Test",
1778
- )
1779
-
1780
- assert requester.max_retries == 0
1781
-
1782
-
1783
- def test_create_page_increment():
1784
- model = PageIncrementModel(
1785
- type="PageIncrement",
1786
- page_size=10,
1787
- start_from_page=1,
1788
- inject_on_first_request=True,
1789
- )
1790
- expected_strategy = PageIncrement(page_size=10, start_from_page=1, inject_on_first_request=True, parameters={}, config=input_config)
1791
-
1792
- strategy = factory.create_page_increment(model, input_config)
1793
-
1794
- assert strategy.page_size == expected_strategy.page_size
1795
- assert strategy.start_from_page == expected_strategy.start_from_page
1796
- assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request
1797
-
1798
-
1799
- def test_create_page_increment_with_interpolated_page_size():
1800
- model = PageIncrementModel(
1801
- type="PageIncrement",
1802
- page_size="{{ config['page_size'] }}",
1803
- start_from_page=1,
1804
- inject_on_first_request=True,
1805
- )
1806
- config = {
1807
- **input_config,
1808
- "page_size": 5
1809
- }
1810
- expected_strategy = PageIncrement(page_size=5, start_from_page=1, inject_on_first_request=True, parameters={}, config=config)
1811
-
1812
- strategy = factory.create_page_increment(model, config)
1813
-
1814
- assert strategy.get_page_size() == expected_strategy.get_page_size()
1815
- assert strategy.start_from_page == expected_strategy.start_from_page
1816
- assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request
1817
-
1818
-
1819
- def test_create_offset_increment():
1820
- model = OffsetIncrementModel(
1821
- type="OffsetIncrement",
1822
- page_size=10,
1823
- inject_on_first_request=True,
1824
- )
1825
- expected_strategy = OffsetIncrement(page_size=10, inject_on_first_request=True, parameters={}, config=input_config)
1826
-
1827
- strategy = factory.create_offset_increment(model, input_config)
1828
-
1829
- assert strategy.page_size == expected_strategy.page_size
1830
- assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request
1831
- assert strategy.config == input_config
1832
-
1833
-
1834
- class MyCustomSchemaLoader(SchemaLoader):
1835
- def get_json_schema(self) -> Mapping[str, Any]:
1836
- """Returns a mapping describing the stream's schema"""
1837
- return {}
1838
-
1839
-
1840
- def test_create_custom_schema_loader():
1841
-
1842
- definition = {
1843
- "type": "CustomSchemaLoader",
1844
- "class_name": "unit_tests.sources.declarative.parsers.test_model_to_component_factory.MyCustomSchemaLoader"
1845
- }
1846
- component = factory.create_component(CustomSchemaLoaderModel, definition, {})
1847
- assert isinstance(component, MyCustomSchemaLoader)