airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (518) hide show
  1. airbyte_cdk/__init__.py +355 -6
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +29 -10
  7. airbyte_cdk/connector.py +24 -24
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
  10. airbyte_cdk/connector_builder/main.py +45 -13
  11. airbyte_cdk/connector_builder/message_grouper.py +189 -50
  12. airbyte_cdk/connector_builder/models.py +3 -2
  13. airbyte_cdk/destinations/__init__.py +4 -3
  14. airbyte_cdk/destinations/destination.py +54 -20
  15. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  16. airbyte_cdk/destinations/vector_db_based/config.py +40 -17
  17. airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
  18. airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
  19. airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
  20. airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
  21. airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
  22. airbyte_cdk/entrypoint.py +153 -44
  23. airbyte_cdk/exception_handler.py +21 -3
  24. airbyte_cdk/logger.py +30 -44
  25. airbyte_cdk/models/__init__.py +13 -2
  26. airbyte_cdk/models/airbyte_protocol.py +86 -1
  27. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  28. airbyte_cdk/models/file_transfer_record_message.py +13 -0
  29. airbyte_cdk/models/well_known_types.py +1 -1
  30. airbyte_cdk/sources/__init__.py +5 -1
  31. airbyte_cdk/sources/abstract_source.py +125 -79
  32. airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
  33. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
  34. airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
  35. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
  36. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  37. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
  38. airbyte_cdk/sources/config.py +3 -2
  39. airbyte_cdk/sources/connector_state_manager.py +49 -83
  40. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  41. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
  42. airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
  43. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  44. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  45. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  46. airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
  47. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
  48. airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
  49. airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
  50. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
  51. airbyte_cdk/sources/declarative/auth/token.py +28 -10
  52. airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
  53. airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
  54. airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
  55. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  56. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  57. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
  58. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
  59. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
  60. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1185 -85
  61. airbyte_cdk/sources/declarative/declarative_source.py +5 -2
  62. airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
  63. airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
  64. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
  65. airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
  66. airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
  67. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  68. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  69. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  70. airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
  71. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
  72. airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
  73. airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
  74. airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
  75. airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
  76. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
  77. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  78. airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
  79. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +346 -0
  80. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
  81. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  82. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
  83. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +173 -74
  84. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  85. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  86. airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
  87. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
  88. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
  89. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
  90. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
  91. airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
  92. airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
  93. airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
  94. airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
  95. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  96. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  97. airbyte_cdk/sources/declarative/models/__init__.py +1 -1
  98. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1319 -603
  99. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
  100. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
  101. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
  102. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1759 -225
  103. airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
  104. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  105. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  106. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
  107. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  108. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
  109. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
  110. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
  111. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
  112. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
  113. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
  114. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
  115. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
  116. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
  117. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
  118. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
  119. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
  120. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  121. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
  122. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
  123. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
  124. airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
  125. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
  126. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
  127. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
  128. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
  129. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
  130. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
  131. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
  132. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
  133. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
  134. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
  135. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
  136. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
  137. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  138. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
  139. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
  140. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
  141. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
  142. airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
  143. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  144. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  145. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  146. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  147. airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
  148. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
  149. airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
  150. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
  151. airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
  152. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
  153. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
  154. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
  155. airbyte_cdk/sources/declarative/spec/spec.py +12 -5
  156. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
  157. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
  158. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
  159. airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
  160. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  161. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  162. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  163. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  164. airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
  165. airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
  166. airbyte_cdk/sources/declarative/types.py +19 -110
  167. airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
  168. airbyte_cdk/sources/embedded/base_integration.py +16 -5
  169. airbyte_cdk/sources/embedded/catalog.py +16 -4
  170. airbyte_cdk/sources/embedded/runner.py +19 -3
  171. airbyte_cdk/sources/embedded/tools.py +5 -2
  172. airbyte_cdk/sources/file_based/README.md +152 -0
  173. airbyte_cdk/sources/file_based/__init__.py +24 -0
  174. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
  175. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
  176. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
  177. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
  178. airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
  179. airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
  180. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  181. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
  182. airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
  183. airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
  184. airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
  185. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
  186. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
  187. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
  188. airbyte_cdk/sources/file_based/exceptions.py +18 -15
  189. airbyte_cdk/sources/file_based/file_based_source.py +140 -33
  190. airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
  191. airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
  192. airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
  193. airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
  194. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  195. airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
  196. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
  197. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
  198. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
  199. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
  200. airbyte_cdk/sources/file_based/remote_file.py +1 -1
  201. airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
  202. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
  203. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
  204. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
  205. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
  206. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
  207. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
  208. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
  209. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
  210. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
  211. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
  212. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
  213. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
  214. airbyte_cdk/sources/http_logger.py +8 -3
  215. airbyte_cdk/sources/message/__init__.py +7 -1
  216. airbyte_cdk/sources/message/repository.py +18 -4
  217. airbyte_cdk/sources/source.py +42 -38
  218. airbyte_cdk/sources/streams/__init__.py +2 -2
  219. airbyte_cdk/sources/streams/availability_strategy.py +54 -3
  220. airbyte_cdk/sources/streams/call_rate.py +64 -21
  221. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  222. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  223. airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
  224. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  225. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  226. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  227. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  228. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
  229. airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
  230. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
  231. airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
  232. airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
  233. airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
  234. airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
  235. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
  236. airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
  237. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
  238. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  239. airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
  240. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
  241. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
  242. airbyte_cdk/sources/streams/core.py +412 -87
  243. airbyte_cdk/sources/streams/http/__init__.py +2 -1
  244. airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
  245. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  246. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  247. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  248. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  249. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  250. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  251. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  252. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  253. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  254. airbyte_cdk/sources/streams/http/exceptions.py +27 -7
  255. airbyte_cdk/sources/streams/http/http.py +369 -246
  256. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  257. airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
  258. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
  259. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
  260. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
  261. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
  262. airbyte_cdk/sources/types.py +154 -0
  263. airbyte_cdk/sources/utils/record_helper.py +36 -21
  264. airbyte_cdk/sources/utils/schema_helpers.py +13 -6
  265. airbyte_cdk/sources/utils/slice_logger.py +4 -1
  266. airbyte_cdk/sources/utils/transform.py +54 -20
  267. airbyte_cdk/sql/_util/hashing.py +34 -0
  268. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  269. airbyte_cdk/sql/constants.py +32 -0
  270. airbyte_cdk/sql/exceptions.py +235 -0
  271. airbyte_cdk/sql/secrets.py +123 -0
  272. airbyte_cdk/sql/shared/__init__.py +15 -0
  273. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  274. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  275. airbyte_cdk/sql/types.py +160 -0
  276. airbyte_cdk/test/catalog_builder.py +70 -18
  277. airbyte_cdk/test/entrypoint_wrapper.py +117 -42
  278. airbyte_cdk/test/mock_http/__init__.py +1 -1
  279. airbyte_cdk/test/mock_http/matcher.py +6 -0
  280. airbyte_cdk/test/mock_http/mocker.py +57 -10
  281. airbyte_cdk/test/mock_http/request.py +19 -3
  282. airbyte_cdk/test/mock_http/response.py +3 -1
  283. airbyte_cdk/test/mock_http/response_builder.py +32 -16
  284. airbyte_cdk/test/state_builder.py +18 -10
  285. airbyte_cdk/test/utils/__init__.py +1 -0
  286. airbyte_cdk/test/utils/data.py +24 -0
  287. airbyte_cdk/test/utils/http_mocking.py +16 -0
  288. airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
  289. airbyte_cdk/test/utils/reading.py +26 -0
  290. airbyte_cdk/utils/__init__.py +2 -1
  291. airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
  292. airbyte_cdk/utils/analytics_message.py +10 -2
  293. airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
  294. airbyte_cdk/utils/event_timing.py +10 -10
  295. airbyte_cdk/utils/mapping_helpers.py +3 -1
  296. airbyte_cdk/utils/message_utils.py +20 -11
  297. airbyte_cdk/utils/print_buffer.py +75 -0
  298. airbyte_cdk/utils/schema_inferrer.py +198 -28
  299. airbyte_cdk/utils/slice_hasher.py +30 -0
  300. airbyte_cdk/utils/spec_schema_transformations.py +6 -3
  301. airbyte_cdk/utils/stream_status_utils.py +8 -1
  302. airbyte_cdk/utils/traced_exception.py +61 -21
  303. airbyte_cdk-6.17.1.dev0.dist-info/METADATA +109 -0
  304. airbyte_cdk-6.17.1.dev0.dist-info/RECORD +350 -0
  305. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/WHEEL +1 -2
  306. airbyte_cdk-6.17.1.dev0.dist-info/entry_points.txt +3 -0
  307. airbyte_cdk/sources/declarative/create_partial.py +0 -92
  308. airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
  309. airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
  310. airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
  311. airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
  312. airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
  313. airbyte_cdk/sources/deprecated/base_source.py +0 -94
  314. airbyte_cdk/sources/deprecated/client.py +0 -99
  315. airbyte_cdk/sources/singer/__init__.py +0 -8
  316. airbyte_cdk/sources/singer/singer_helpers.py +0 -304
  317. airbyte_cdk/sources/singer/source.py +0 -186
  318. airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
  319. airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
  320. airbyte_cdk/sources/streams/http/auth/core.py +0 -29
  321. airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
  322. airbyte_cdk/sources/streams/http/auth/token.py +0 -47
  323. airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
  324. airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
  325. airbyte_cdk/sources/utils/schema_models.py +0 -84
  326. airbyte_cdk-0.72.1.dist-info/METADATA +0 -243
  327. airbyte_cdk-0.72.1.dist-info/RECORD +0 -466
  328. airbyte_cdk-0.72.1.dist-info/top_level.txt +0 -3
  329. source_declarative_manifest/main.py +0 -29
  330. unit_tests/connector_builder/__init__.py +0 -3
  331. unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
  332. unit_tests/connector_builder/test_message_grouper.py +0 -713
  333. unit_tests/connector_builder/utils.py +0 -27
  334. unit_tests/destinations/test_destination.py +0 -243
  335. unit_tests/singer/test_singer_helpers.py +0 -56
  336. unit_tests/singer/test_singer_source.py +0 -112
  337. unit_tests/sources/__init__.py +0 -0
  338. unit_tests/sources/concurrent_source/__init__.py +0 -3
  339. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
  340. unit_tests/sources/declarative/__init__.py +0 -3
  341. unit_tests/sources/declarative/auth/__init__.py +0 -3
  342. unit_tests/sources/declarative/auth/test_oauth.py +0 -331
  343. unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
  344. unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
  345. unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
  346. unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
  347. unit_tests/sources/declarative/checks/__init__.py +0 -3
  348. unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
  349. unit_tests/sources/declarative/decoders/__init__.py +0 -0
  350. unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
  351. unit_tests/sources/declarative/external_component.py +0 -13
  352. unit_tests/sources/declarative/extractors/__init__.py +0 -3
  353. unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
  354. unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
  355. unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
  356. unit_tests/sources/declarative/incremental/__init__.py +0 -0
  357. unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
  358. unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
  359. unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
  360. unit_tests/sources/declarative/interpolation/__init__.py +0 -3
  361. unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
  362. unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
  363. unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
  364. unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
  365. unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
  366. unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
  367. unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
  368. unit_tests/sources/declarative/parsers/__init__.py +0 -3
  369. unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
  370. unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
  371. unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1847
  372. unit_tests/sources/declarative/parsers/testing_components.py +0 -36
  373. unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
  374. unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
  375. unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
  376. unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
  377. unit_tests/sources/declarative/requesters/__init__.py +0 -3
  378. unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
  379. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
  380. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
  381. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
  382. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
  383. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
  384. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
  385. unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
  386. unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
  387. unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
  388. unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
  389. unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
  390. unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
  391. unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
  392. unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
  393. unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
  394. unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
  395. unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
  396. unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
  397. unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
  398. unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
  399. unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
  400. unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
  401. unit_tests/sources/declarative/retrievers/__init__.py +0 -3
  402. unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
  403. unit_tests/sources/declarative/schema/__init__.py +0 -6
  404. unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
  405. unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
  406. unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
  407. unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
  408. unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
  409. unit_tests/sources/declarative/states/__init__.py +0 -3
  410. unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
  411. unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
  412. unit_tests/sources/declarative/test_create_partial.py +0 -83
  413. unit_tests/sources/declarative/test_declarative_stream.py +0 -103
  414. unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
  415. unit_tests/sources/declarative/test_types.py +0 -39
  416. unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
  417. unit_tests/sources/file_based/__init__.py +0 -0
  418. unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
  419. unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
  420. unit_tests/sources/file_based/config/__init__.py +0 -0
  421. unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
  422. unit_tests/sources/file_based/config/test_csv_format.py +0 -34
  423. unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
  424. unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
  425. unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
  426. unit_tests/sources/file_based/file_types/__init__.py +0 -0
  427. unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
  428. unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
  429. unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
  430. unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
  431. unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
  432. unit_tests/sources/file_based/helpers.py +0 -70
  433. unit_tests/sources/file_based/in_memory_files_source.py +0 -211
  434. unit_tests/sources/file_based/scenarios/__init__.py +0 -0
  435. unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
  436. unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
  437. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
  438. unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
  439. unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
  440. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
  441. unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
  442. unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
  443. unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
  444. unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
  445. unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
  446. unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
  447. unit_tests/sources/file_based/stream/__init__.py +0 -0
  448. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  449. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
  450. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
  451. unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
  452. unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
  453. unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
  454. unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
  455. unit_tests/sources/file_based/test_scenarios.py +0 -253
  456. unit_tests/sources/file_based/test_schema_helpers.py +0 -346
  457. unit_tests/sources/fixtures/__init__.py +0 -3
  458. unit_tests/sources/fixtures/source_test_fixture.py +0 -153
  459. unit_tests/sources/message/__init__.py +0 -0
  460. unit_tests/sources/message/test_repository.py +0 -153
  461. unit_tests/sources/streams/__init__.py +0 -0
  462. unit_tests/sources/streams/concurrent/__init__.py +0 -3
  463. unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
  464. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
  465. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
  466. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
  467. unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
  468. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
  469. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
  470. unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
  471. unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
  472. unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
  473. unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
  474. unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
  475. unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
  476. unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
  477. unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
  478. unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
  479. unit_tests/sources/streams/http/__init__.py +0 -0
  480. unit_tests/sources/streams/http/auth/__init__.py +0 -0
  481. unit_tests/sources/streams/http/auth/test_auth.py +0 -173
  482. unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
  483. unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
  484. unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
  485. unit_tests/sources/streams/http/test_http.py +0 -635
  486. unit_tests/sources/streams/test_availability_strategy.py +0 -70
  487. unit_tests/sources/streams/test_call_rate.py +0 -300
  488. unit_tests/sources/streams/test_stream_read.py +0 -405
  489. unit_tests/sources/streams/test_streams_core.py +0 -184
  490. unit_tests/sources/test_abstract_source.py +0 -1442
  491. unit_tests/sources/test_concurrent_source.py +0 -112
  492. unit_tests/sources/test_config.py +0 -92
  493. unit_tests/sources/test_connector_state_manager.py +0 -482
  494. unit_tests/sources/test_http_logger.py +0 -252
  495. unit_tests/sources/test_integration_source.py +0 -86
  496. unit_tests/sources/test_source.py +0 -684
  497. unit_tests/sources/test_source_read.py +0 -460
  498. unit_tests/test/__init__.py +0 -0
  499. unit_tests/test/mock_http/__init__.py +0 -0
  500. unit_tests/test/mock_http/test_matcher.py +0 -53
  501. unit_tests/test/mock_http/test_mocker.py +0 -214
  502. unit_tests/test/mock_http/test_request.py +0 -117
  503. unit_tests/test/mock_http/test_response_builder.py +0 -177
  504. unit_tests/test/test_entrypoint_wrapper.py +0 -240
  505. unit_tests/utils/__init__.py +0 -0
  506. unit_tests/utils/test_datetime_format_inferrer.py +0 -60
  507. unit_tests/utils/test_mapping_helpers.py +0 -54
  508. unit_tests/utils/test_message_utils.py +0 -91
  509. unit_tests/utils/test_rate_limiting.py +0 -26
  510. unit_tests/utils/test_schema_inferrer.py +0 -202
  511. unit_tests/utils/test_secret_utils.py +0 -135
  512. unit_tests/utils/test_stream_status_utils.py +0 -61
  513. unit_tests/utils/test_traced_exception.py +0 -107
  514. /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
  515. {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
  516. {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
  517. {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
  518. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/LICENSE.txt +0 -0
@@ -1,32 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- import pytest as pytest
6
- from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
7
- from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_input_provider import InterpolatedRequestInputProvider
8
-
9
-
10
- @pytest.mark.parametrize(
11
- "test_name, input_request_data, expected_request_data",
12
- [
13
- ("test_static_map_data", {"a_static_request_param": "a_static_value"}, {"a_static_request_param": "a_static_value"}),
14
- ("test_map_depends_on_stream_slice", {"read_from_slice": "{{ stream_slice['slice_key'] }}"}, {"read_from_slice": "slice_value"}),
15
- ("test_map_depends_on_config", {"read_from_config": "{{ config['config_key'] }}"}, {"read_from_config": "value_of_config"}),
16
- (
17
- "test_map_depends_on_parameters",
18
- {"read_from_parameters": "{{ parameters['read_from_parameters'] }}"},
19
- {"read_from_parameters": "value_of_parameters"},
20
- ),
21
- ("test_defaults_to_empty_dictionary", None, {}),
22
- ],
23
- )
24
- def test_initialize_interpolated_mapping_request_input_provider(test_name, input_request_data, expected_request_data):
25
- config = {"config_key": "value_of_config"}
26
- stream_slice = {"slice_key": "slice_value"}
27
- parameters = {"read_from_parameters": "value_of_parameters"}
28
- provider = InterpolatedRequestInputProvider(request_inputs=input_request_data, config=config, parameters=parameters)
29
- actual_request_data = provider.eval_request_inputs(stream_state={}, stream_slice=stream_slice)
30
-
31
- assert isinstance(provider._interpolator, InterpolatedMapping)
32
- assert actual_request_data == expected_request_data
@@ -1,3 +0,0 @@
1
- #
2
- # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3
- #
@@ -1,542 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
- from unittest.mock import MagicMock, Mock, patch
5
-
6
- import pytest
7
- import requests
8
- from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode, Type
9
- from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth
10
- from airbyte_cdk.sources.declarative.incremental import Cursor, DatetimeBasedCursor
11
- from airbyte_cdk.sources.declarative.partition_routers import SinglePartitionRouter
12
- from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus
13
- from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType
14
- from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
15
- from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever, SimpleRetrieverTestReadDecorator
16
- from airbyte_cdk.sources.declarative.types import Record
17
-
18
- A_SLICE_STATE = {"slice_state": "slice state value"}
19
- A_STREAM_SLICE = {"stream slice": "slice value"}
20
- A_STREAM_STATE = {"stream state": "state value"}
21
-
22
- primary_key = "pk"
23
- records = [{"id": 1}, {"id": 2}]
24
- request_response_logs = [
25
- AirbyteLogMessage(level=Level.INFO, message="request:{}"),
26
- AirbyteLogMessage(level=Level.INFO, message="response{}"),
27
- ]
28
- config = {}
29
-
30
-
31
- @patch.object(SimpleRetriever, "_read_pages", return_value=iter([]))
32
- def test_simple_retriever_full(mock_http_stream):
33
- requester = MagicMock()
34
- request_params = {"param": "value"}
35
- requester.get_request_params.return_value = request_params
36
-
37
- paginator = MagicMock()
38
- next_page_token = {"cursor": "cursor_value"}
39
- paginator.path.return_value = None
40
- paginator.next_page_token.return_value = next_page_token
41
- paginator.get_requesyyt_headers.return_value = {}
42
-
43
- record_selector = MagicMock()
44
- record_selector.select_records.return_value = records
45
-
46
- cursor = MagicMock(spec=Cursor)
47
- stream_slices = [{"date": "2022-01-01"}, {"date": "2022-01-02"}]
48
- cursor.stream_slices.return_value = stream_slices
49
-
50
- response = requests.Response()
51
- response.status_code = 200
52
-
53
- underlying_state = {"date": "2021-01-01"}
54
- cursor.get_stream_state.return_value = underlying_state
55
-
56
- requester.get_authenticator.return_value = NoAuth({})
57
- url_base = "https://airbyte.io"
58
- requester.get_url_base.return_value = url_base
59
- path = "/v1"
60
- requester.get_path.return_value = path
61
- http_method = HttpMethod.GET
62
- requester.get_method.return_value = http_method
63
- backoff_time = 60
64
- should_retry = ResponseStatus.retry(backoff_time)
65
- requester.interpret_response_status.return_value = should_retry
66
- request_body_json = {"body": "json"}
67
- requester.request_body_json.return_value = request_body_json
68
-
69
- request_body_data = {"body": "data"}
70
- requester.get_request_body_data.return_value = request_body_data
71
- request_body_json = {"body": "json"}
72
- requester.get_request_body_json.return_value = request_body_json
73
- request_kwargs = {"kwarg": "value"}
74
- requester.request_kwargs.return_value = request_kwargs
75
-
76
- retriever = SimpleRetriever(
77
- name="stream_name",
78
- primary_key=primary_key,
79
- requester=requester,
80
- paginator=paginator,
81
- record_selector=record_selector,
82
- stream_slicer=cursor,
83
- cursor=cursor,
84
- parameters={},
85
- config={},
86
- )
87
-
88
- assert retriever.primary_key == primary_key
89
- assert retriever.state == underlying_state
90
- assert retriever._next_page_token(response) == next_page_token
91
- assert retriever._request_params(None, None, None) == {}
92
- assert retriever.stream_slices() == stream_slices
93
-
94
- assert retriever._last_response is None
95
- assert retriever._records_from_last_response == []
96
- assert retriever._parse_response(response, stream_state={}, records_schema={}) == records
97
- assert retriever._last_response == response
98
- assert retriever._records_from_last_response == records
99
-
100
- [r for r in retriever.read_records(SyncMode.full_refresh)]
101
- paginator.reset.assert_called()
102
-
103
-
104
- @patch.object(SimpleRetriever, "_read_pages", return_value=iter([*request_response_logs, *records]))
105
- def test_simple_retriever_with_request_response_logs(mock_http_stream):
106
- requester = MagicMock()
107
- paginator = MagicMock()
108
- record_selector = MagicMock()
109
- stream_slicer = DatetimeBasedCursor(
110
- start_datetime="",
111
- end_datetime="",
112
- step="P1D",
113
- cursor_field="id",
114
- datetime_format="",
115
- cursor_granularity="P1D",
116
- config={},
117
- parameters={},
118
- )
119
-
120
- retriever = SimpleRetriever(
121
- name="stream_name",
122
- primary_key=primary_key,
123
- requester=requester,
124
- paginator=paginator,
125
- record_selector=record_selector,
126
- stream_slicer=stream_slicer,
127
- parameters={},
128
- config={},
129
- )
130
-
131
- actual_messages = [r for r in retriever.read_records(SyncMode.full_refresh)]
132
- paginator.reset.assert_called()
133
-
134
- assert isinstance(actual_messages[0], AirbyteLogMessage)
135
- assert isinstance(actual_messages[1], AirbyteLogMessage)
136
- assert actual_messages[2] == records[0]
137
- assert actual_messages[3] == records[1]
138
-
139
-
140
- @patch.object(SimpleRetriever, "_read_pages", return_value=iter([]))
141
- def test_simple_retriever_with_request_response_log_last_records(mock_http_stream):
142
- requester = MagicMock()
143
- paginator = MagicMock()
144
- record_selector = MagicMock()
145
- record_selector.select_records.return_value = request_response_logs
146
- response = requests.Response()
147
- response.status_code = 200
148
- stream_slicer = DatetimeBasedCursor(
149
- start_datetime="",
150
- end_datetime="",
151
- step="P1D",
152
- cursor_field="id",
153
- datetime_format="",
154
- cursor_granularity="P1D",
155
- config={},
156
- parameters={},
157
- )
158
-
159
- retriever = SimpleRetriever(
160
- name="stream_name",
161
- primary_key=primary_key,
162
- requester=requester,
163
- paginator=paginator,
164
- record_selector=record_selector,
165
- stream_slicer=stream_slicer,
166
- parameters={},
167
- config={},
168
- )
169
-
170
- assert retriever._last_response is None
171
- assert retriever._records_from_last_response == []
172
- assert retriever._parse_response(response, stream_state={}, records_schema={}) == request_response_logs
173
- assert retriever._last_response == response
174
- assert retriever._records_from_last_response == request_response_logs
175
-
176
- [r for r in retriever.read_records(SyncMode.full_refresh)]
177
- paginator.reset.assert_called()
178
-
179
-
180
- @pytest.mark.parametrize(
181
- "test_name, paginator_mapping, stream_slicer_mapping, expected_mapping",
182
- [
183
- ("test_empty_headers", {}, {}, {}),
184
- ("test_header_from_pagination_and_slicer", {"offset": 1000}, {"key": "value"}, {"key": "value", "offset": 1000}),
185
- ("test_header_from_stream_slicer", {}, {"slice": "slice_value"}, {"slice": "slice_value"}),
186
- ("test_duplicate_header_slicer_paginator", {"k": "v"}, {"k": "slice_value"}, None),
187
- ],
188
- )
189
- def test_get_request_options_from_pagination(test_name, paginator_mapping, stream_slicer_mapping, expected_mapping):
190
- # This test does not test request headers because they must be strings
191
- paginator = MagicMock()
192
- paginator.get_request_params.return_value = paginator_mapping
193
- paginator.get_request_body_data.return_value = paginator_mapping
194
- paginator.get_request_body_json.return_value = paginator_mapping
195
-
196
- stream_slicer = MagicMock()
197
- stream_slicer.get_request_params.return_value = stream_slicer_mapping
198
- stream_slicer.get_request_body_data.return_value = stream_slicer_mapping
199
- stream_slicer.get_request_body_json.return_value = stream_slicer_mapping
200
-
201
- record_selector = MagicMock()
202
- retriever = SimpleRetriever(
203
- name="stream_name",
204
- primary_key=primary_key,
205
- requester=MagicMock(),
206
- record_selector=record_selector,
207
- paginator=paginator,
208
- stream_slicer=stream_slicer,
209
- parameters={},
210
- config={},
211
- )
212
-
213
- request_option_type_to_method = {
214
- RequestOptionType.request_parameter: retriever._request_params,
215
- RequestOptionType.body_data: retriever._request_body_data,
216
- RequestOptionType.body_json: retriever._request_body_json,
217
- }
218
-
219
- for _, method in request_option_type_to_method.items():
220
- if expected_mapping is not None:
221
- actual_mapping = method(None, None, None)
222
- assert expected_mapping == actual_mapping
223
- else:
224
- try:
225
- method(None, None, None)
226
- assert False
227
- except ValueError:
228
- pass
229
-
230
-
231
- @pytest.mark.parametrize(
232
- "test_name, paginator_mapping, expected_mapping",
233
- [
234
- ("test_only_base_headers", {}, {"key": "value"}),
235
- ("test_header_from_pagination", {"offset": 1000}, {"key": "value", "offset": "1000"}),
236
- ("test_duplicate_header", {"key": 1000}, None),
237
- ],
238
- )
239
- def test_get_request_headers(test_name, paginator_mapping, expected_mapping):
240
- # This test is separate from the other request options because request headers must be strings
241
- paginator = MagicMock()
242
- paginator.get_request_headers.return_value = paginator_mapping
243
- requester = MagicMock(use_cache=False)
244
-
245
- stream_slicer = MagicMock()
246
- stream_slicer.get_request_headers.return_value = {"key": "value"}
247
-
248
- record_selector = MagicMock()
249
- retriever = SimpleRetriever(
250
- name="stream_name",
251
- primary_key=primary_key,
252
- requester=requester,
253
- record_selector=record_selector,
254
- stream_slicer=stream_slicer,
255
- paginator=paginator,
256
- parameters={},
257
- config={},
258
- )
259
-
260
- request_option_type_to_method = {
261
- RequestOptionType.header: retriever._request_headers,
262
- }
263
-
264
- for _, method in request_option_type_to_method.items():
265
- if expected_mapping:
266
- actual_mapping = method(None, None, None)
267
- assert expected_mapping == actual_mapping
268
- else:
269
- try:
270
- method(None, None, None)
271
- assert False
272
- except ValueError:
273
- pass
274
-
275
-
276
- @pytest.mark.parametrize(
277
- "test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping",
278
- [
279
- ("test_do_not_ignore_stream_slicer_params_if_ignore_is_true_but_no_next_page_token", {"key_from_pagination": "1000"}, True, None, {"key_from_pagination": "1000"}),
280
- ("test_do_not_ignore_stream_slicer_params_if_ignore_is_false_and_no_next_page_token", {"key_from_pagination": "1000"}, False, None, {"key_from_pagination": "1000", "key_from_slicer": "value"}),
281
- ("test_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, True, {"page": 2}, {"key_from_pagination": "1000"}),
282
- ("test_do_not_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, False, {"page": 2}, {"key_from_pagination": "1000", "key_from_slicer": "value"}),
283
- ],
284
- )
285
- def test_ignore_stream_slicer_parameters_on_paginated_requests(test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping):
286
- # This test is separate from the other request options because request headers must be strings
287
- paginator = MagicMock()
288
- paginator.get_request_headers.return_value = paginator_mapping
289
- requester = MagicMock(use_cache=False)
290
-
291
- stream_slicer = MagicMock()
292
- stream_slicer.get_request_headers.return_value = {"key_from_slicer": "value"}
293
-
294
- record_selector = MagicMock()
295
- retriever = SimpleRetriever(
296
- name="stream_name",
297
- primary_key=primary_key,
298
- requester=requester,
299
- record_selector=record_selector,
300
- stream_slicer=stream_slicer,
301
- paginator=paginator,
302
- ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests,
303
- parameters={},
304
- config={},
305
- )
306
-
307
- request_option_type_to_method = {
308
- RequestOptionType.header: retriever._request_headers,
309
- }
310
-
311
- for _, method in request_option_type_to_method.items():
312
- actual_mapping = method(None, None, next_page_token={"next_page_token": "1000"})
313
- assert expected_mapping == actual_mapping
314
-
315
-
316
- @pytest.mark.parametrize(
317
- "test_name, slicer_body_data, paginator_body_data, expected_body_data",
318
- [
319
- ("test_only_slicer_mapping", {"key": "value"}, {}, {"key": "value"}),
320
- ("test_only_slicer_string", "key=value", {}, "key=value"),
321
- ("test_slicer_mapping_and_paginator_no_duplicate", {"key": "value"}, {"offset": 1000}, {"key": "value", "offset": 1000}),
322
- ("test_slicer_mapping_and_paginator_with_duplicate", {"key": "value"}, {"key": 1000}, None),
323
- ("test_slicer_string_and_paginator", "key=value", {"offset": 1000}, None),
324
- ],
325
- )
326
- def test_request_body_data(test_name, slicer_body_data, paginator_body_data, expected_body_data):
327
- paginator = MagicMock()
328
- paginator.get_request_body_data.return_value = paginator_body_data
329
- requester = MagicMock(use_cache=False)
330
-
331
- stream_slicer = MagicMock()
332
- stream_slicer.get_request_body_data.return_value = slicer_body_data
333
-
334
- record_selector = MagicMock()
335
- retriever = SimpleRetriever(
336
- name="stream_name",
337
- primary_key=primary_key,
338
- requester=requester,
339
- record_selector=record_selector,
340
- paginator=paginator,
341
- stream_slicer=stream_slicer,
342
- parameters={},
343
- config={},
344
- )
345
-
346
- if expected_body_data:
347
- actual_body_data = retriever._request_body_data(None, None, None)
348
- assert expected_body_data == actual_body_data
349
- else:
350
- try:
351
- retriever._request_body_data(None, None, None)
352
- assert False
353
- except ValueError:
354
- pass
355
-
356
-
357
- @pytest.mark.parametrize(
358
- "test_name, requester_path, paginator_path, expected_path",
359
- [
360
- ("test_path_from_requester", "/v1/path", None, None),
361
- ("test_path_from_paginator", "/v1/path/", "/v2/paginator", "/v2/paginator"),
362
- ],
363
- )
364
- def test_path(test_name, requester_path, paginator_path, expected_path):
365
- paginator = MagicMock()
366
- paginator.path.return_value = paginator_path
367
- requester = MagicMock(use_cache=False)
368
-
369
- requester.get_path.return_value = requester_path
370
-
371
- record_selector = MagicMock()
372
- retriever = SimpleRetriever(
373
- name="stream_name",
374
- primary_key=primary_key,
375
- requester=requester,
376
- record_selector=record_selector,
377
- paginator=paginator,
378
- parameters={},
379
- config={},
380
- )
381
-
382
- actual_path = retriever._paginator_path()
383
- assert expected_path == actual_path
384
-
385
-
386
- def test_limit_stream_slices():
387
- maximum_number_of_slices = 4
388
- stream_slicer = MagicMock()
389
- stream_slicer.stream_slices.return_value = _generate_slices(maximum_number_of_slices * 2)
390
- retriever = SimpleRetrieverTestReadDecorator(
391
- name="stream_name",
392
- primary_key=primary_key,
393
- requester=MagicMock(),
394
- paginator=MagicMock(),
395
- record_selector=MagicMock(),
396
- stream_slicer=stream_slicer,
397
- maximum_number_of_slices=maximum_number_of_slices,
398
- parameters={},
399
- config={},
400
- )
401
-
402
- truncated_slices = list(retriever.stream_slices())
403
-
404
- assert truncated_slices == _generate_slices(maximum_number_of_slices)
405
-
406
-
407
- @pytest.mark.parametrize(
408
- "test_name, first_greater_than_second",
409
- [
410
- ("test_first_greater_than_second", True),
411
- ("test_second_greater_than_first", False),
412
- ],
413
- )
414
- def test_when_read_records_then_cursor_close_slice_with_greater_record(test_name, first_greater_than_second):
415
- first_record = Record({"first": 1}, {})
416
- second_record = Record({"second": 2}, {})
417
- records = [first_record, second_record]
418
- record_selector = MagicMock()
419
- record_selector.select_records.return_value = records
420
- cursor = MagicMock(spec=Cursor)
421
- cursor.is_greater_than_or_equal.return_value = first_greater_than_second
422
- paginator = MagicMock()
423
- paginator.get_request_headers.return_value = {}
424
-
425
- retriever = SimpleRetriever(
426
- name="stream_name",
427
- primary_key=primary_key,
428
- requester=MagicMock(),
429
- paginator=paginator,
430
- record_selector=record_selector,
431
- stream_slicer=cursor,
432
- cursor=cursor,
433
- parameters={},
434
- config={},
435
- )
436
- stream_slice = {"repository": "airbyte"}
437
-
438
- def retriever_read_pages(_, __, ___):
439
- return retriever._parse_records(response=MagicMock(), stream_state={}, stream_slice=stream_slice, records_schema={})
440
-
441
- with patch.object(
442
- SimpleRetriever,
443
- "_read_pages",
444
- return_value=iter([first_record, second_record]),
445
- side_effect=retriever_read_pages,
446
- ):
447
- list(retriever.read_records(stream_slice=stream_slice, records_schema={}))
448
- cursor.close_slice.assert_called_once_with(stream_slice, first_record if first_greater_than_second else second_record)
449
-
450
-
451
- def test_given_stream_data_is_not_record_when_read_records_then_update_slice_with_optional_record():
452
- stream_data = [AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="a log message"))]
453
- record_selector = MagicMock()
454
- record_selector.select_records.return_value = []
455
- cursor = MagicMock(spec=Cursor)
456
-
457
- retriever = SimpleRetriever(
458
- name="stream_name",
459
- primary_key=primary_key,
460
- requester=MagicMock(),
461
- paginator=Mock(),
462
- record_selector=record_selector,
463
- stream_slicer=cursor,
464
- cursor=cursor,
465
- parameters={},
466
- config={},
467
- )
468
- stream_slice = {"repository": "airbyte"}
469
-
470
- def retriever_read_pages(_, __, ___):
471
- return retriever._parse_records(response=MagicMock(), stream_state={}, stream_slice=stream_slice, records_schema={})
472
-
473
- with patch.object(
474
- SimpleRetriever,
475
- "_read_pages",
476
- return_value=iter(stream_data),
477
- side_effect=retriever_read_pages,
478
- ):
479
- list(retriever.read_records(stream_slice=stream_slice, records_schema={}))
480
- cursor.observe.assert_not_called()
481
- cursor.close_slice.assert_called_once_with(stream_slice, None)
482
-
483
-
484
- def _generate_slices(number_of_slices):
485
- return [{"date": f"2022-01-0{day + 1}"} for day in range(number_of_slices)]
486
-
487
-
488
- @patch.object(SimpleRetriever, "_read_pages", return_value=iter([]))
489
- def test_given_state_selector_when_read_records_use_stream_state(http_stream_read_pages, mocker):
490
- requester = MagicMock()
491
- paginator = MagicMock()
492
- record_selector = MagicMock()
493
- cursor = MagicMock(spec=Cursor)
494
- cursor.select_state = MagicMock(return_value=A_SLICE_STATE)
495
- cursor.get_stream_state = MagicMock(return_value=A_STREAM_STATE)
496
-
497
- retriever = SimpleRetriever(
498
- name="stream_name",
499
- primary_key=primary_key,
500
- requester=requester,
501
- paginator=paginator,
502
- record_selector=record_selector,
503
- stream_slicer=cursor,
504
- cursor=cursor,
505
- parameters={},
506
- config={},
507
- )
508
-
509
- list(retriever.read_records(stream_slice=A_STREAM_SLICE, records_schema={}))
510
-
511
- http_stream_read_pages.assert_called_once_with(mocker.ANY, A_STREAM_STATE, A_STREAM_SLICE)
512
-
513
-
514
- def test_emit_log_request_response_messages(mocker):
515
- record_selector = MagicMock()
516
- record_selector.select_records.return_value = records
517
-
518
- request = requests.PreparedRequest()
519
- request.headers = {"header": "value"}
520
- request.url = "http://byrde.enterprises.com/casinos"
521
-
522
- response = requests.Response()
523
- response.request = request
524
- response.status_code = 200
525
-
526
- format_http_message_mock = mocker.patch("airbyte_cdk.sources.declarative.retrievers.simple_retriever.format_http_message")
527
- requester = MagicMock()
528
- retriever = SimpleRetrieverTestReadDecorator(
529
- name="stream_name",
530
- primary_key=primary_key,
531
- requester=requester,
532
- paginator=MagicMock(),
533
- record_selector=record_selector,
534
- stream_slicer=SinglePartitionRouter(parameters={}),
535
- parameters={},
536
- config={},
537
- )
538
-
539
- retriever._fetch_next_page(stream_state={}, stream_slice={})
540
-
541
- assert requester.send_request.call_args_list[0][1]["log_formatter"] is not None
542
- assert requester.send_request.call_args_list[0][1]["log_formatter"](response) == format_http_message_mock.return_value
@@ -1,6 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
- from .source_test.SourceTest import SourceTest
5
-
6
- __all__ = ["SourceTest"]
@@ -1,8 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
-
6
- class SourceTest:
7
- def __init__(self):
8
- pass
@@ -1,3 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
@@ -1,32 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- from unittest.mock import MagicMock
6
-
7
- import pytest
8
- from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader
9
-
10
-
11
- @pytest.mark.parametrize(
12
- "found_schema, found_error, expected_schema",
13
- [
14
- pytest.param(
15
- {"type": "object", "properties": {}}, None, {"type": "object", "properties": {}}, id="test_has_schema_in_default_location"
16
- ),
17
- pytest.param(None, FileNotFoundError, {}, id="test_schema_file_does_not_exist"),
18
- ],
19
- )
20
- def test_get_json_schema(found_schema, found_error, expected_schema):
21
- default_schema_loader = DefaultSchemaLoader({}, {})
22
-
23
- json_file_schema_loader = MagicMock()
24
- if found_schema:
25
- json_file_schema_loader.get_json_schema.return_value = {"type": "object", "properties": {}}
26
- if found_error:
27
- json_file_schema_loader.get_json_schema.side_effect = found_error
28
-
29
- default_schema_loader.default_loader = json_file_schema_loader
30
-
31
- actual_schema = default_schema_loader.get_json_schema()
32
- assert actual_schema == expected_schema
@@ -1,19 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- import pytest
6
- from airbyte_cdk.sources.declarative.schema import InlineSchemaLoader
7
-
8
-
9
- @pytest.mark.parametrize(
10
- "test_name, input_schema, expected_schema",
11
- [
12
- ("schema", {"k": "string"}, {"k": "string"}),
13
- ("empty_schema", {}, {}),
14
- ],
15
- )
16
- def test_static_schema_loads(test_name, input_schema, expected_schema):
17
- schema_loader = InlineSchemaLoader(input_schema, {})
18
-
19
- assert schema_loader.get_json_schema() == expected_schema
@@ -1,26 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- import pytest
6
- from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader
7
-
8
-
9
- @pytest.mark.parametrize(
10
- "test_name, input_path, expected_resource, expected_path",
11
- [
12
- ("path_prefixed_with_dot", "./source_example/schemas/lists.json", "source_example", "schemas/lists.json"),
13
- ("path_prefixed_with_slash", "/source_example/schemas/lists.json", "source_example", "schemas/lists.json"),
14
- ("path_starting_with_source", "source_example/schemas/lists.json", "source_example", "schemas/lists.json"),
15
- ("path_starting_missing_source", "schemas/lists.json", "schemas", "lists.json"),
16
- ("path_with_file_only", "lists.json", "", "lists.json"),
17
- ("empty_path_does_not_crash", "", "", ""),
18
- ("empty_path_with_slash_does_not_crash", "/", "", ""),
19
- ],
20
- )
21
- def test_extract_resource_and_schema_path(test_name, input_path, expected_resource, expected_path):
22
- json_schema = JsonFileSchemaLoader({}, {}, input_path)
23
- actual_resource, actual_path = json_schema.extract_resource_and_schema_path(input_path)
24
-
25
- assert actual_resource == expected_resource
26
- assert actual_path == expected_path
@@ -1,3 +0,0 @@
1
- #
2
- # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3
- #