airbyte-cdk 0.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (368) hide show
  1. airbyte_cdk/__init__.py +358 -0
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +236 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +104 -0
  7. airbyte_cdk/connector.py +123 -0
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/__init__.py +3 -0
  10. airbyte_cdk/connector_builder/connector_builder_handler.py +121 -0
  11. airbyte_cdk/connector_builder/main.py +107 -0
  12. airbyte_cdk/connector_builder/models.py +73 -0
  13. airbyte_cdk/connector_builder/test_reader/__init__.py +7 -0
  14. airbyte_cdk/connector_builder/test_reader/helpers.py +689 -0
  15. airbyte_cdk/connector_builder/test_reader/message_grouper.py +173 -0
  16. airbyte_cdk/connector_builder/test_reader/reader.py +441 -0
  17. airbyte_cdk/connector_builder/test_reader/types.py +83 -0
  18. airbyte_cdk/destinations/__init__.py +8 -0
  19. airbyte_cdk/destinations/destination.py +154 -0
  20. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  21. airbyte_cdk/destinations/vector_db_based/__init__.py +38 -0
  22. airbyte_cdk/destinations/vector_db_based/config.py +298 -0
  23. airbyte_cdk/destinations/vector_db_based/document_processor.py +223 -0
  24. airbyte_cdk/destinations/vector_db_based/embedder.py +303 -0
  25. airbyte_cdk/destinations/vector_db_based/indexer.py +78 -0
  26. airbyte_cdk/destinations/vector_db_based/test_utils.py +63 -0
  27. airbyte_cdk/destinations/vector_db_based/utils.py +35 -0
  28. airbyte_cdk/destinations/vector_db_based/writer.py +104 -0
  29. airbyte_cdk/entrypoint.py +414 -0
  30. airbyte_cdk/exception_handler.py +56 -0
  31. airbyte_cdk/logger.py +109 -0
  32. airbyte_cdk/models/__init__.py +72 -0
  33. airbyte_cdk/models/airbyte_protocol.py +88 -0
  34. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  35. airbyte_cdk/models/well_known_types.py +5 -0
  36. airbyte_cdk/py.typed +0 -0
  37. airbyte_cdk/sources/__init__.py +26 -0
  38. airbyte_cdk/sources/abstract_source.py +326 -0
  39. airbyte_cdk/sources/concurrent_source/__init__.py +8 -0
  40. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +255 -0
  41. airbyte_cdk/sources/concurrent_source/concurrent_source.py +165 -0
  42. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +147 -0
  43. airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py +24 -0
  44. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  45. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +115 -0
  46. airbyte_cdk/sources/config.py +27 -0
  47. airbyte_cdk/sources/connector_state_manager.py +161 -0
  48. airbyte_cdk/sources/declarative/__init__.py +3 -0
  49. airbyte_cdk/sources/declarative/async_job/__init__.py +0 -0
  50. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  51. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +525 -0
  52. airbyte_cdk/sources/declarative/async_job/job_tracker.py +79 -0
  53. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  54. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  55. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  56. airbyte_cdk/sources/declarative/auth/__init__.py +8 -0
  57. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +42 -0
  58. airbyte_cdk/sources/declarative/auth/jwt.py +197 -0
  59. airbyte_cdk/sources/declarative/auth/oauth.py +293 -0
  60. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +45 -0
  61. airbyte_cdk/sources/declarative/auth/token.py +267 -0
  62. airbyte_cdk/sources/declarative/auth/token_provider.py +82 -0
  63. airbyte_cdk/sources/declarative/checks/__init__.py +24 -0
  64. airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +61 -0
  65. airbyte_cdk/sources/declarative/checks/check_stream.py +56 -0
  66. airbyte_cdk/sources/declarative/checks/connection_checker.py +35 -0
  67. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  68. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  69. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +526 -0
  70. airbyte_cdk/sources/declarative/datetime/__init__.py +3 -0
  71. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +65 -0
  72. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +118 -0
  73. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +3975 -0
  74. airbyte_cdk/sources/declarative/declarative_source.py +36 -0
  75. airbyte_cdk/sources/declarative/declarative_stream.py +241 -0
  76. airbyte_cdk/sources/declarative/decoders/__init__.py +33 -0
  77. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +218 -0
  78. airbyte_cdk/sources/declarative/decoders/decoder.py +32 -0
  79. airbyte_cdk/sources/declarative/decoders/decoder_parser.py +30 -0
  80. airbyte_cdk/sources/declarative/decoders/json_decoder.py +65 -0
  81. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  82. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  83. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  84. airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py +56 -0
  85. airbyte_cdk/sources/declarative/exceptions.py +9 -0
  86. airbyte_cdk/sources/declarative/extractors/__init__.py +21 -0
  87. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +86 -0
  88. airbyte_cdk/sources/declarative/extractors/http_selector.py +37 -0
  89. airbyte_cdk/sources/declarative/extractors/record_extractor.py +27 -0
  90. airbyte_cdk/sources/declarative/extractors/record_filter.py +91 -0
  91. airbyte_cdk/sources/declarative/extractors/record_selector.py +170 -0
  92. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +176 -0
  93. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  94. airbyte_cdk/sources/declarative/incremental/__init__.py +37 -0
  95. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +497 -0
  96. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +459 -0
  97. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  98. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +357 -0
  99. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +380 -0
  100. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  101. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  102. airbyte_cdk/sources/declarative/interpolation/__init__.py +9 -0
  103. airbyte_cdk/sources/declarative/interpolation/filters.py +139 -0
  104. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +66 -0
  105. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +56 -0
  106. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +52 -0
  107. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +79 -0
  108. airbyte_cdk/sources/declarative/interpolation/interpolation.py +34 -0
  109. airbyte_cdk/sources/declarative/interpolation/jinja.py +161 -0
  110. airbyte_cdk/sources/declarative/interpolation/macros.py +191 -0
  111. airbyte_cdk/sources/declarative/manifest_declarative_source.py +421 -0
  112. airbyte_cdk/sources/declarative/migrations/__init__.py +0 -0
  113. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  114. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  115. airbyte_cdk/sources/declarative/models/__init__.py +2 -0
  116. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2503 -0
  117. airbyte_cdk/sources/declarative/parsers/__init__.py +3 -0
  118. airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +157 -0
  119. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +21 -0
  120. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +172 -0
  121. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +213 -0
  122. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +3407 -0
  123. airbyte_cdk/sources/declarative/partition_routers/__init__.py +29 -0
  124. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  125. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  126. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +121 -0
  127. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  128. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +63 -0
  129. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +437 -0
  130. airbyte_cdk/sources/declarative/requesters/README.md +56 -0
  131. airbyte_cdk/sources/declarative/requesters/__init__.py +9 -0
  132. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +25 -0
  133. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +23 -0
  134. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +45 -0
  135. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +45 -0
  136. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +41 -0
  137. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +70 -0
  138. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +77 -0
  139. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +17 -0
  140. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +101 -0
  141. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +147 -0
  142. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  143. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +17 -0
  144. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +179 -0
  145. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +350 -0
  146. airbyte_cdk/sources/declarative/requesters/http_requester.py +433 -0
  147. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +21 -0
  148. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +327 -0
  149. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +76 -0
  150. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +65 -0
  151. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +25 -0
  152. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +98 -0
  153. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +102 -0
  154. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +71 -0
  155. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +48 -0
  156. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +66 -0
  157. airbyte_cdk/sources/declarative/requesters/request_option.py +117 -0
  158. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +23 -0
  159. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +92 -0
  160. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  161. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +59 -0
  162. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +68 -0
  163. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +119 -0
  164. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +79 -0
  165. airbyte_cdk/sources/declarative/requesters/request_path.py +15 -0
  166. airbyte_cdk/sources/declarative/requesters/requester.py +144 -0
  167. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  168. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  169. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  170. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  171. airbyte_cdk/sources/declarative/retrievers/__init__.py +19 -0
  172. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +124 -0
  173. airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
  174. airbyte_cdk/sources/declarative/retrievers/retriever.py +54 -0
  175. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +702 -0
  176. airbyte_cdk/sources/declarative/schema/__init__.py +25 -0
  177. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +47 -0
  178. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +285 -0
  179. airbyte_cdk/sources/declarative/schema/inline_schema_loader.py +19 -0
  180. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +92 -0
  181. airbyte_cdk/sources/declarative/schema/schema_loader.py +17 -0
  182. airbyte_cdk/sources/declarative/spec/__init__.py +7 -0
  183. airbyte_cdk/sources/declarative/spec/spec.py +48 -0
  184. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +7 -0
  185. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +93 -0
  186. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +25 -0
  187. airbyte_cdk/sources/declarative/transformations/__init__.py +17 -0
  188. airbyte_cdk/sources/declarative/transformations/add_fields.py +146 -0
  189. airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py +61 -0
  190. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  191. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  192. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  193. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  194. airbyte_cdk/sources/declarative/transformations/remove_fields.py +75 -0
  195. airbyte_cdk/sources/declarative/transformations/transformation.py +37 -0
  196. airbyte_cdk/sources/declarative/types.py +25 -0
  197. airbyte_cdk/sources/declarative/yaml_declarative_source.py +67 -0
  198. airbyte_cdk/sources/file_based/README.md +152 -0
  199. airbyte_cdk/sources/file_based/__init__.py +24 -0
  200. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +11 -0
  201. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +73 -0
  202. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +149 -0
  203. airbyte_cdk/sources/file_based/config/__init__.py +0 -0
  204. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +153 -0
  205. airbyte_cdk/sources/file_based/config/avro_format.py +25 -0
  206. airbyte_cdk/sources/file_based/config/csv_format.py +210 -0
  207. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  208. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +99 -0
  209. airbyte_cdk/sources/file_based/config/jsonl_format.py +18 -0
  210. airbyte_cdk/sources/file_based/config/parquet_format.py +25 -0
  211. airbyte_cdk/sources/file_based/config/unstructured_format.py +102 -0
  212. airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py +81 -0
  213. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +8 -0
  214. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +21 -0
  215. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +33 -0
  216. airbyte_cdk/sources/file_based/exceptions.py +159 -0
  217. airbyte_cdk/sources/file_based/file_based_source.py +466 -0
  218. airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py +123 -0
  219. airbyte_cdk/sources/file_based/file_based_stream_reader.py +209 -0
  220. airbyte_cdk/sources/file_based/file_record_data.py +22 -0
  221. airbyte_cdk/sources/file_based/file_types/__init__.py +37 -0
  222. airbyte_cdk/sources/file_based/file_types/avro_parser.py +233 -0
  223. airbyte_cdk/sources/file_based/file_types/csv_parser.py +527 -0
  224. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  225. airbyte_cdk/sources/file_based/file_types/file_transfer.py +30 -0
  226. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +86 -0
  227. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +145 -0
  228. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +275 -0
  229. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +480 -0
  230. airbyte_cdk/sources/file_based/remote_file.py +18 -0
  231. airbyte_cdk/sources/file_based/schema_helpers.py +281 -0
  232. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +17 -0
  233. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +20 -0
  234. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +52 -0
  235. airbyte_cdk/sources/file_based/stream/__init__.py +13 -0
  236. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +197 -0
  237. airbyte_cdk/sources/file_based/stream/concurrent/__init__.py +0 -0
  238. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +343 -0
  239. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +9 -0
  240. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +59 -0
  241. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +313 -0
  242. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +83 -0
  243. airbyte_cdk/sources/file_based/stream/cursor/__init__.py +4 -0
  244. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +66 -0
  245. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +149 -0
  246. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +396 -0
  247. airbyte_cdk/sources/file_based/stream/identities_stream.py +49 -0
  248. airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +92 -0
  249. airbyte_cdk/sources/file_based/types.py +10 -0
  250. airbyte_cdk/sources/http_config.py +10 -0
  251. airbyte_cdk/sources/http_logger.py +55 -0
  252. airbyte_cdk/sources/message/__init__.py +19 -0
  253. airbyte_cdk/sources/message/repository.py +137 -0
  254. airbyte_cdk/sources/source.py +95 -0
  255. airbyte_cdk/sources/specs/transfer_modes.py +26 -0
  256. airbyte_cdk/sources/streams/__init__.py +8 -0
  257. airbyte_cdk/sources/streams/availability_strategy.py +84 -0
  258. airbyte_cdk/sources/streams/call_rate.py +704 -0
  259. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  260. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  261. airbyte_cdk/sources/streams/checkpoint/cursor.py +77 -0
  262. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  263. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  264. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  265. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  266. airbyte_cdk/sources/streams/concurrent/__init__.py +3 -0
  267. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +96 -0
  268. airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py +37 -0
  269. airbyte_cdk/sources/streams/concurrent/adapters.py +397 -0
  270. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +94 -0
  271. airbyte_cdk/sources/streams/concurrent/clamping.py +99 -0
  272. airbyte_cdk/sources/streams/concurrent/cursor.py +481 -0
  273. airbyte_cdk/sources/streams/concurrent/cursor_types.py +32 -0
  274. airbyte_cdk/sources/streams/concurrent/default_stream.py +102 -0
  275. airbyte_cdk/sources/streams/concurrent/exceptions.py +18 -0
  276. airbyte_cdk/sources/streams/concurrent/helpers.py +42 -0
  277. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +64 -0
  278. airbyte_cdk/sources/streams/concurrent/partition_reader.py +45 -0
  279. airbyte_cdk/sources/streams/concurrent/partitions/__init__.py +3 -0
  280. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +48 -0
  281. airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py +18 -0
  282. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  283. airbyte_cdk/sources/streams/concurrent/partitions/types.py +38 -0
  284. airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py +0 -0
  285. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +182 -0
  286. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +223 -0
  287. airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
  288. airbyte_cdk/sources/streams/core.py +703 -0
  289. airbyte_cdk/sources/streams/http/__init__.py +10 -0
  290. airbyte_cdk/sources/streams/http/availability_strategy.py +54 -0
  291. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  292. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  293. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  294. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  295. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  296. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  297. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  298. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  299. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  300. airbyte_cdk/sources/streams/http/exceptions.py +61 -0
  301. airbyte_cdk/sources/streams/http/http.py +673 -0
  302. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  303. airbyte_cdk/sources/streams/http/rate_limiting.py +158 -0
  304. airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py +14 -0
  305. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +479 -0
  306. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +34 -0
  307. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +436 -0
  308. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +83 -0
  309. airbyte_cdk/sources/streams/permissions/identities_stream.py +75 -0
  310. airbyte_cdk/sources/streams/utils/__init__.py +3 -0
  311. airbyte_cdk/sources/types.py +169 -0
  312. airbyte_cdk/sources/utils/__init__.py +7 -0
  313. airbyte_cdk/sources/utils/casing.py +12 -0
  314. airbyte_cdk/sources/utils/files_directory.py +15 -0
  315. airbyte_cdk/sources/utils/record_helper.py +53 -0
  316. airbyte_cdk/sources/utils/schema_helpers.py +230 -0
  317. airbyte_cdk/sources/utils/slice_logger.py +57 -0
  318. airbyte_cdk/sources/utils/transform.py +277 -0
  319. airbyte_cdk/sources/utils/types.py +7 -0
  320. airbyte_cdk/sql/__init__.py +0 -0
  321. airbyte_cdk/sql/_util/__init__.py +0 -0
  322. airbyte_cdk/sql/_util/hashing.py +34 -0
  323. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  324. airbyte_cdk/sql/constants.py +32 -0
  325. airbyte_cdk/sql/exceptions.py +235 -0
  326. airbyte_cdk/sql/secrets.py +123 -0
  327. airbyte_cdk/sql/shared/__init__.py +15 -0
  328. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  329. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  330. airbyte_cdk/sql/types.py +160 -0
  331. airbyte_cdk/test/__init__.py +7 -0
  332. airbyte_cdk/test/catalog_builder.py +81 -0
  333. airbyte_cdk/test/entrypoint_wrapper.py +250 -0
  334. airbyte_cdk/test/mock_http/__init__.py +6 -0
  335. airbyte_cdk/test/mock_http/matcher.py +41 -0
  336. airbyte_cdk/test/mock_http/mocker.py +185 -0
  337. airbyte_cdk/test/mock_http/request.py +103 -0
  338. airbyte_cdk/test/mock_http/response.py +28 -0
  339. airbyte_cdk/test/mock_http/response_builder.py +237 -0
  340. airbyte_cdk/test/state_builder.py +33 -0
  341. airbyte_cdk/test/utils/__init__.py +1 -0
  342. airbyte_cdk/test/utils/data.py +24 -0
  343. airbyte_cdk/test/utils/http_mocking.py +16 -0
  344. airbyte_cdk/test/utils/manifest_only_fixtures.py +59 -0
  345. airbyte_cdk/test/utils/reading.py +26 -0
  346. airbyte_cdk/utils/__init__.py +10 -0
  347. airbyte_cdk/utils/airbyte_secrets_utils.py +80 -0
  348. airbyte_cdk/utils/analytics_message.py +25 -0
  349. airbyte_cdk/utils/constants.py +5 -0
  350. airbyte_cdk/utils/datetime_format_inferrer.py +94 -0
  351. airbyte_cdk/utils/datetime_helpers.py +499 -0
  352. airbyte_cdk/utils/event_timing.py +85 -0
  353. airbyte_cdk/utils/is_cloud_environment.py +18 -0
  354. airbyte_cdk/utils/mapping_helpers.py +162 -0
  355. airbyte_cdk/utils/message_utils.py +26 -0
  356. airbyte_cdk/utils/oneof_option_config.py +33 -0
  357. airbyte_cdk/utils/print_buffer.py +75 -0
  358. airbyte_cdk/utils/schema_inferrer.py +270 -0
  359. airbyte_cdk/utils/slice_hasher.py +37 -0
  360. airbyte_cdk/utils/spec_schema_transformations.py +26 -0
  361. airbyte_cdk/utils/stream_status_utils.py +43 -0
  362. airbyte_cdk/utils/traced_exception.py +145 -0
  363. airbyte_cdk-0.0.0.dev0.dist-info/LICENSE.txt +19 -0
  364. airbyte_cdk-0.0.0.dev0.dist-info/LICENSE_SHORT +1 -0
  365. airbyte_cdk-0.0.0.dev0.dist-info/METADATA +111 -0
  366. airbyte_cdk-0.0.0.dev0.dist-info/RECORD +368 -0
  367. airbyte_cdk-0.0.0.dev0.dist-info/WHEEL +4 -0
  368. airbyte_cdk-0.0.0.dev0.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,185 @@
1
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
+
3
+ import contextlib
4
+ import functools
5
+ from collections import defaultdict
6
+ from enum import Enum
7
+ from types import TracebackType
8
+ from typing import Callable, Dict, Iterable, List, Optional, Union
9
+
10
+ import requests_mock
11
+
12
+ from airbyte_cdk.test.mock_http.matcher import HttpRequestMatcher
13
+ from airbyte_cdk.test.mock_http.request import HttpRequest
14
+ from airbyte_cdk.test.mock_http.response import HttpResponse
15
+
16
+
17
+ class SupportedHttpMethods(str, Enum):
18
+ GET = "get"
19
+ PATCH = "patch"
20
+ POST = "post"
21
+ PUT = "put"
22
+ DELETE = "delete"
23
+
24
+
25
+ class HttpMocker(contextlib.ContextDecorator):
26
+ """
27
+ WARNING 1: This implementation only works if the lib used to perform HTTP requests is `requests`.
28
+
29
+ WARNING 2: Given multiple requests that are not mutually exclusive, the request will match the first one. This can happen in scenarios
30
+ where the same request is added twice (in which case there will always be an exception because we will never match the second
31
+ request) or in a case like this:
32
+ ```
33
+ http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1", "more_granular": "2"}), <...>)
34
+ http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1"}), <...>)
35
+ requests.get(_A_URL, headers={"less_granular": "1", "more_granular": "2"})
36
+ ```
37
+ In the example above, the matcher would match the second mock as requests_mock iterate over the matcher in reverse order (see
38
+ https://github.com/jamielennox/requests-mock/blob/c06f124a33f56e9f03840518e19669ba41b93202/requests_mock/adapter.py#L246) even
39
+ though the request sent is a better match for the first `http_mocker.get`.
40
+ """
41
+
42
+ def __init__(self) -> None:
43
+ self._mocker = requests_mock.Mocker()
44
+ self._matchers: Dict[SupportedHttpMethods, List[HttpRequestMatcher]] = defaultdict(list)
45
+
46
+ def __enter__(self) -> "HttpMocker":
47
+ self._mocker.__enter__()
48
+ return self
49
+
50
+ def __exit__(
51
+ self,
52
+ exc_type: Optional[BaseException],
53
+ exc_val: Optional[BaseException],
54
+ exc_tb: Optional[TracebackType],
55
+ ) -> None:
56
+ self._mocker.__exit__(exc_type, exc_val, exc_tb)
57
+
58
+ def _validate_all_matchers_called(self) -> None:
59
+ for matcher in self._get_matchers():
60
+ if not matcher.has_expected_match_count():
61
+ raise ValueError(f"Invalid number of matches for `{matcher}`")
62
+
63
+ def _mock_request_method(
64
+ self,
65
+ method: SupportedHttpMethods,
66
+ request: HttpRequest,
67
+ responses: Union[HttpResponse, List[HttpResponse]],
68
+ ) -> None:
69
+ if isinstance(responses, HttpResponse):
70
+ responses = [responses]
71
+
72
+ matcher = HttpRequestMatcher(request, len(responses))
73
+ if matcher in self._matchers[method]:
74
+ raise ValueError(f"Request {matcher.request} already mocked")
75
+ self._matchers[method].append(matcher)
76
+
77
+ getattr(self._mocker, method)(
78
+ requests_mock.ANY,
79
+ additional_matcher=self._matches_wrapper(matcher),
80
+ response_list=[
81
+ {
82
+ self._get_body_field(response): response.body,
83
+ "status_code": response.status_code,
84
+ "headers": response.headers,
85
+ }
86
+ for response in responses
87
+ ],
88
+ )
89
+
90
+ @staticmethod
91
+ def _get_body_field(response: HttpResponse) -> str:
92
+ return "text" if isinstance(response.body, str) else "content"
93
+
94
+ def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
95
+ self._mock_request_method(SupportedHttpMethods.GET, request, responses)
96
+
97
+ def patch(
98
+ self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
99
+ ) -> None:
100
+ self._mock_request_method(SupportedHttpMethods.PATCH, request, responses)
101
+
102
+ def post(
103
+ self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
104
+ ) -> None:
105
+ self._mock_request_method(SupportedHttpMethods.POST, request, responses)
106
+
107
+ def put(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
108
+ self._mock_request_method(SupportedHttpMethods.PUT, request, responses)
109
+
110
+ def delete(
111
+ self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
112
+ ) -> None:
113
+ self._mock_request_method(SupportedHttpMethods.DELETE, request, responses)
114
+
115
+ @staticmethod
116
+ def _matches_wrapper(
117
+ matcher: HttpRequestMatcher,
118
+ ) -> Callable[[requests_mock.request._RequestObjectProxy], bool]:
119
+ def matches(requests_mock_request: requests_mock.request._RequestObjectProxy) -> bool:
120
+ # query_params are provided as part of `requests_mock_request.url`
121
+ http_request = HttpRequest(
122
+ requests_mock_request.url,
123
+ query_params={},
124
+ headers=requests_mock_request.headers,
125
+ body=requests_mock_request.body,
126
+ )
127
+ return matcher.matches(http_request)
128
+
129
+ return matches
130
+
131
+ def assert_number_of_calls(self, request: HttpRequest, number_of_calls: int) -> None:
132
+ corresponding_matchers = list(
133
+ filter(lambda matcher: matcher.request is request, self._get_matchers())
134
+ )
135
+ if len(corresponding_matchers) != 1:
136
+ raise ValueError(
137
+ f"Was expecting only one matcher to match the request but got `{corresponding_matchers}`"
138
+ )
139
+
140
+ assert corresponding_matchers[0].actual_number_of_matches == number_of_calls
141
+
142
+ # trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"`
143
+ def __call__(self, f): # type: ignore
144
+ @functools.wraps(f)
145
+ def wrapper(*args, **kwargs): # type: ignore # this is a very generic wrapper that does not need to be typed
146
+ with self:
147
+ assertion_error = None
148
+
149
+ kwargs["http_mocker"] = self
150
+ try:
151
+ result = f(*args, **kwargs)
152
+ except requests_mock.NoMockAddress as no_mock_exception:
153
+ matchers_as_string = "\n\t".join(
154
+ map(lambda matcher: str(matcher.request), self._get_matchers())
155
+ )
156
+ raise ValueError(
157
+ f"No matcher matches {no_mock_exception.args[0]} with headers `{no_mock_exception.request.headers}` "
158
+ f"and body `{no_mock_exception.request.body}`. "
159
+ f"Matchers currently configured are:\n\t{matchers_as_string}."
160
+ ) from no_mock_exception
161
+ except AssertionError as test_assertion:
162
+ assertion_error = test_assertion
163
+
164
+ # We validate the matchers before raising the assertion error because we want to show the tester if an HTTP request wasn't
165
+ # mocked correctly
166
+ try:
167
+ self._validate_all_matchers_called()
168
+ except ValueError as http_mocker_exception:
169
+ # This seems useless as it catches ValueError and raises ValueError but without this, the prevailing error message in
170
+ # the output is the function call that failed the assertion, whereas raising `ValueError(http_mocker_exception)`
171
+ # like we do here provides additional context for the exception.
172
+ raise ValueError(http_mocker_exception) from None
173
+ if assertion_error:
174
+ raise assertion_error
175
+ return result
176
+
177
+ return wrapper
178
+
179
+ def _get_matchers(self) -> Iterable[HttpRequestMatcher]:
180
+ for matchers in self._matchers.values():
181
+ yield from matchers
182
+
183
+ def clear_all_matchers(self) -> None:
184
+ """Clears all stored matchers by resetting the _matchers list to an empty state."""
185
+ self._matchers = defaultdict(list)
@@ -0,0 +1,103 @@
1
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
+
3
+ import json
4
+ from typing import Any, List, Mapping, Optional, Union
5
+ from urllib.parse import parse_qs, urlencode, urlparse
6
+
7
+ ANY_QUERY_PARAMS = "any query_parameters"
8
+
9
+
10
+ def _is_subdict(small: Mapping[str, str], big: Mapping[str, str]) -> bool:
11
+ return dict(big, **small) == big
12
+
13
+
14
+ class HttpRequest:
15
+ def __init__(
16
+ self,
17
+ url: str,
18
+ query_params: Optional[Union[str, Mapping[str, Union[str, List[str]]]]] = None,
19
+ headers: Optional[Mapping[str, str]] = None,
20
+ body: Optional[Union[str, bytes, Mapping[str, Any]]] = None,
21
+ ) -> None:
22
+ self._parsed_url = urlparse(url)
23
+ self._query_params = query_params
24
+ if not self._parsed_url.query and query_params:
25
+ self._parsed_url = urlparse(f"{url}?{self._encode_qs(query_params)}")
26
+ elif self._parsed_url.query and query_params:
27
+ raise ValueError(
28
+ "If query params are provided as part of the url, `query_params` should be empty"
29
+ )
30
+
31
+ self._headers = headers or {}
32
+ self._body = body
33
+
34
+ @staticmethod
35
+ def _encode_qs(query_params: Union[str, Mapping[str, Union[str, List[str]]]]) -> str:
36
+ if isinstance(query_params, str):
37
+ return query_params
38
+ return urlencode(query_params, doseq=True)
39
+
40
+ def matches(self, other: Any) -> bool:
41
+ """
42
+ If the body of any request is a Mapping, we compare as Mappings which means that the order is not important.
43
+ If the body is a string, encoding ISO-8859-1 will be assumed
44
+ Headers only need to be a subset of `other` in order to match
45
+ """
46
+ if isinstance(other, HttpRequest):
47
+ # if `other` is a mapping, we match as an object and formatting is not considers
48
+ if isinstance(self._body, Mapping) or isinstance(other._body, Mapping):
49
+ body_match = self._to_mapping(self._body) == self._to_mapping(other._body)
50
+ else:
51
+ body_match = self._to_bytes(self._body) == self._to_bytes(other._body)
52
+
53
+ return (
54
+ self._parsed_url.scheme == other._parsed_url.scheme
55
+ and self._parsed_url.hostname == other._parsed_url.hostname
56
+ and self._parsed_url.path == other._parsed_url.path
57
+ and (
58
+ ANY_QUERY_PARAMS in (self._query_params, other._query_params)
59
+ or parse_qs(self._parsed_url.query) == parse_qs(other._parsed_url.query)
60
+ )
61
+ and _is_subdict(other._headers, self._headers)
62
+ and body_match
63
+ )
64
+ return False
65
+
66
+ @staticmethod
67
+ def _to_mapping(
68
+ body: Optional[Union[str, bytes, Mapping[str, Any]]],
69
+ ) -> Optional[Mapping[str, Any]]:
70
+ if isinstance(body, Mapping):
71
+ return body
72
+ elif isinstance(body, bytes):
73
+ return json.loads(body.decode()) # type: ignore # assumes return type of Mapping[str, Any]
74
+ elif isinstance(body, str):
75
+ return json.loads(body) # type: ignore # assumes return type of Mapping[str, Any]
76
+ return None
77
+
78
+ @staticmethod
79
+ def _to_bytes(body: Optional[Union[str, bytes]]) -> bytes:
80
+ if isinstance(body, bytes):
81
+ return body
82
+ elif isinstance(body, str):
83
+ # `ISO-8859-1` is the default encoding used by requests
84
+ return body.encode("ISO-8859-1")
85
+ return b""
86
+
87
+ def __str__(self) -> str:
88
+ return f"{self._parsed_url} with headers {self._headers} and body {self._body!r})"
89
+
90
+ def __repr__(self) -> str:
91
+ return (
92
+ f"HttpRequest(request={self._parsed_url}, headers={self._headers}, body={self._body!r})"
93
+ )
94
+
95
+ def __eq__(self, other: Any) -> bool:
96
+ if isinstance(other, HttpRequest):
97
+ return (
98
+ self._parsed_url == other._parsed_url
99
+ and self._query_params == other._query_params
100
+ and self._headers == other._headers
101
+ and self._body == other._body
102
+ )
103
+ return False
@@ -0,0 +1,28 @@
1
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
+
3
+ from types import MappingProxyType
4
+ from typing import Mapping, Union
5
+
6
+
7
+ class HttpResponse:
8
+ def __init__(
9
+ self,
10
+ body: Union[str, bytes],
11
+ status_code: int = 200,
12
+ headers: Mapping[str, str] = MappingProxyType({}),
13
+ ):
14
+ self._body = body
15
+ self._status_code = status_code
16
+ self._headers = headers
17
+
18
+ @property
19
+ def body(self) -> Union[str, bytes]:
20
+ return self._body
21
+
22
+ @property
23
+ def status_code(self) -> int:
24
+ return self._status_code
25
+
26
+ @property
27
+ def headers(self) -> Mapping[str, str]:
28
+ return self._headers
@@ -0,0 +1,237 @@
1
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
+
3
+ import functools
4
+ import json
5
+ from abc import ABC, abstractmethod
6
+ from pathlib import Path as FilePath
7
+ from typing import Any, Dict, List, Optional, Union
8
+
9
+ from airbyte_cdk.test.mock_http.response import HttpResponse
10
+ from airbyte_cdk.test.utils.data import get_unit_test_folder
11
+
12
+
13
+ def _extract(path: List[str], response_template: Dict[str, Any]) -> Any:
14
+ return functools.reduce(lambda a, b: a[b], path, response_template)
15
+
16
+
17
+ def _replace_value(dictionary: Dict[str, Any], path: List[str], value: Any) -> None:
18
+ current = dictionary
19
+ for key in path[:-1]:
20
+ current = current[key]
21
+ current[path[-1]] = value
22
+
23
+
24
+ def _write(dictionary: Dict[str, Any], path: List[str], value: Any) -> None:
25
+ current = dictionary
26
+ for key in path[:-1]:
27
+ current = current.setdefault(key, {})
28
+ current[path[-1]] = value
29
+
30
+
31
+ class Path(ABC):
32
+ @abstractmethod
33
+ def write(self, template: Dict[str, Any], value: Any) -> None:
34
+ pass
35
+
36
+ @abstractmethod
37
+ def update(self, template: Dict[str, Any], value: Any) -> None:
38
+ pass
39
+
40
+ def extract(self, template: Dict[str, Any]) -> Any:
41
+ pass
42
+
43
+
44
+ class FieldPath(Path):
45
+ def __init__(self, field: str):
46
+ self._path = [field]
47
+
48
+ def write(self, template: Dict[str, Any], value: Any) -> None:
49
+ _write(template, self._path, value)
50
+
51
+ def update(self, template: Dict[str, Any], value: Any) -> None:
52
+ _replace_value(template, self._path, value)
53
+
54
+ def extract(self, template: Dict[str, Any]) -> Any:
55
+ return _extract(self._path, template)
56
+
57
+ def __str__(self) -> str:
58
+ return f"FieldPath(field={self._path[0]})"
59
+
60
+
61
+ class NestedPath(Path):
62
+ def __init__(self, path: List[str]):
63
+ self._path = path
64
+
65
+ def write(self, template: Dict[str, Any], value: Any) -> None:
66
+ _write(template, self._path, value)
67
+
68
+ def update(self, template: Dict[str, Any], value: Any) -> None:
69
+ _replace_value(template, self._path, value)
70
+
71
+ def extract(self, template: Dict[str, Any]) -> Any:
72
+ return _extract(self._path, template)
73
+
74
+ def __str__(self) -> str:
75
+ return f"NestedPath(path={self._path})"
76
+
77
+
78
+ class PaginationStrategy(ABC):
79
+ @abstractmethod
80
+ def update(self, response: Dict[str, Any]) -> None:
81
+ pass
82
+
83
+
84
+ class FieldUpdatePaginationStrategy(PaginationStrategy):
85
+ def __init__(self, path: Path, value: Any):
86
+ self._path = path
87
+ self._value = value
88
+
89
+ def update(self, response: Dict[str, Any]) -> None:
90
+ self._path.update(response, self._value)
91
+
92
+
93
+ class RecordBuilder:
94
+ def __init__(
95
+ self,
96
+ template: Dict[str, Any],
97
+ id_path: Optional[Path],
98
+ cursor_path: Optional[Union[FieldPath, NestedPath]],
99
+ ):
100
+ self._record = template
101
+ self._id_path = id_path
102
+ self._cursor_path = cursor_path
103
+
104
+ self._validate_template()
105
+
106
+ def _validate_template(self) -> None:
107
+ paths_to_validate = [
108
+ ("_id_path", self._id_path),
109
+ ("_cursor_path", self._cursor_path),
110
+ ]
111
+ for field_name, field_path in paths_to_validate:
112
+ self._validate_field(field_name, field_path)
113
+
114
+ def _validate_field(self, field_name: str, path: Optional[Path]) -> None:
115
+ try:
116
+ if path and not path.extract(self._record):
117
+ raise ValueError(
118
+ f"{field_name} `{path}` was provided but it is not part of the template `{self._record}`"
119
+ )
120
+ except (IndexError, KeyError) as exception:
121
+ raise ValueError(
122
+ f"{field_name} `{path}` was provided but it is not part of the template `{self._record}`"
123
+ ) from exception
124
+
125
+ def with_id(self, identifier: Any) -> "RecordBuilder":
126
+ self._set_field("id", self._id_path, identifier)
127
+ return self
128
+
129
+ def with_cursor(self, cursor_value: Any) -> "RecordBuilder":
130
+ self._set_field("cursor", self._cursor_path, cursor_value)
131
+ return self
132
+
133
+ def with_field(self, path: Path, value: Any) -> "RecordBuilder":
134
+ path.write(self._record, value)
135
+ return self
136
+
137
+ def _set_field(self, field_name: str, path: Optional[Path], value: Any) -> None:
138
+ if not path:
139
+ raise ValueError(
140
+ f"{field_name}_path was not provided and hence, the record {field_name} can't be modified. Please provide `id_field` while "
141
+ f"instantiating RecordBuilder to leverage this capability"
142
+ )
143
+ path.update(self._record, value)
144
+
145
+ def build(self) -> Dict[str, Any]:
146
+ return self._record
147
+
148
+
149
+ class HttpResponseBuilder:
150
+ def __init__(
151
+ self,
152
+ template: Dict[str, Any],
153
+ records_path: Union[FieldPath, NestedPath],
154
+ pagination_strategy: Optional[PaginationStrategy],
155
+ ):
156
+ self._response = template
157
+ self._records: List[RecordBuilder] = []
158
+ self._records_path = records_path
159
+ self._pagination_strategy = pagination_strategy
160
+ self._status_code = 200
161
+
162
+ def with_record(self, record: RecordBuilder) -> "HttpResponseBuilder":
163
+ self._records.append(record)
164
+ return self
165
+
166
+ def with_pagination(self) -> "HttpResponseBuilder":
167
+ if not self._pagination_strategy:
168
+ raise ValueError(
169
+ "`pagination_strategy` was not provided and hence, fields related to the pagination can't be modified. Please provide "
170
+ "`pagination_strategy` while instantiating ResponseBuilder to leverage this capability"
171
+ )
172
+ self._pagination_strategy.update(self._response)
173
+ return self
174
+
175
+ def with_status_code(self, status_code: int) -> "HttpResponseBuilder":
176
+ self._status_code = status_code
177
+ return self
178
+
179
+ def build(self) -> HttpResponse:
180
+ self._records_path.update(self._response, [record.build() for record in self._records])
181
+ return HttpResponse(json.dumps(self._response), self._status_code)
182
+
183
+
184
+ def _get_unit_test_folder(execution_folder: str) -> FilePath:
185
+ # FIXME: This function should be removed after the next CDK release to avoid breaking amazon-seller-partner test code.
186
+ return get_unit_test_folder(execution_folder)
187
+
188
+
189
+ def find_template(resource: str, execution_folder: str) -> Dict[str, Any]:
190
+ response_template_filepath = str(
191
+ get_unit_test_folder(execution_folder)
192
+ / "resource"
193
+ / "http"
194
+ / "response"
195
+ / f"{resource}.json"
196
+ )
197
+ with open(response_template_filepath, "r") as template_file:
198
+ return json.load(template_file) # type: ignore # we assume the dev correctly set up the resource file
199
+
200
+
201
+ def find_binary_response(resource: str, execution_folder: str) -> bytes:
202
+ response_filepath = str(
203
+ get_unit_test_folder(execution_folder) / "resource" / "http" / "response" / f"{resource}"
204
+ )
205
+ with open(response_filepath, "rb") as response_file:
206
+ return response_file.read() # type: ignore # we assume the dev correctly set up the resource file
207
+
208
+
209
+ def create_record_builder(
210
+ response_template: Dict[str, Any],
211
+ records_path: Union[FieldPath, NestedPath],
212
+ record_id_path: Optional[Path] = None,
213
+ record_cursor_path: Optional[Union[FieldPath, NestedPath]] = None,
214
+ ) -> RecordBuilder:
215
+ """
216
+ This will use the first record define at `records_path` as a template for the records. If more records are defined, they will be ignored
217
+ """
218
+ try:
219
+ record_template = records_path.extract(response_template)[0]
220
+ if not record_template:
221
+ raise ValueError(
222
+ f"Could not extract any record from template at path `{records_path}`. "
223
+ f"Please fix the template to provide a record sample or fix `records_path`."
224
+ )
225
+ return RecordBuilder(record_template, record_id_path, record_cursor_path)
226
+ except (IndexError, KeyError):
227
+ raise ValueError(
228
+ f"Error while extracting records at path `{records_path}` from response template `{response_template}`"
229
+ )
230
+
231
+
232
+ def create_response_builder(
233
+ response_template: Dict[str, Any],
234
+ records_path: Union[FieldPath, NestedPath],
235
+ pagination_strategy: Optional[PaginationStrategy] = None,
236
+ ) -> HttpResponseBuilder:
237
+ return HttpResponseBuilder(response_template, records_path, pagination_strategy)
@@ -0,0 +1,33 @@
1
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
2
+
3
+ from typing import Any, List
4
+
5
+ from airbyte_cdk.models import (
6
+ AirbyteStateBlob,
7
+ AirbyteStateMessage,
8
+ AirbyteStateType,
9
+ AirbyteStreamState,
10
+ StreamDescriptor,
11
+ )
12
+
13
+
14
+ class StateBuilder:
15
+ def __init__(self) -> None:
16
+ self._state: List[AirbyteStateMessage] = []
17
+
18
+ def with_stream_state(self, stream_name: str, state: Any) -> "StateBuilder":
19
+ self._state.append(
20
+ AirbyteStateMessage(
21
+ type=AirbyteStateType.STREAM,
22
+ stream=AirbyteStreamState(
23
+ stream_state=state
24
+ if isinstance(state, AirbyteStateBlob)
25
+ else AirbyteStateBlob(state),
26
+ stream_descriptor=StreamDescriptor(**{"name": stream_name}),
27
+ ),
28
+ )
29
+ )
30
+ return self
31
+
32
+ def build(self) -> List[AirbyteStateMessage]:
33
+ return self._state
@@ -0,0 +1 @@
1
+ # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
@@ -0,0 +1,24 @@
1
+ # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
+
3
+ from pydantic import FilePath
4
+
5
+
6
+ def get_unit_test_folder(execution_folder: str) -> FilePath:
7
+ path = FilePath(execution_folder)
8
+ while path.name != "unit_tests":
9
+ if path.name == path.root or path.name == path.drive:
10
+ raise ValueError(
11
+ f"Could not find `unit_tests` folder as a parent of {execution_folder}"
12
+ )
13
+ path = path.parent
14
+ return path
15
+
16
+
17
+ def read_resource_file_contents(resource: str, test_location: str) -> str:
18
+ """Read the contents of a test data file from the test resource folder."""
19
+ file_path = str(
20
+ get_unit_test_folder(test_location) / "resource" / "http" / "response" / f"{resource}"
21
+ )
22
+ with open(file_path) as f:
23
+ response = f.read()
24
+ return response
@@ -0,0 +1,16 @@
1
+ # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
+
3
+ import re
4
+ from typing import Any, Mapping
5
+
6
+ from requests_mock import Mocker
7
+
8
+
9
+ def register_mock_responses(
10
+ mocker: Mocker, http_calls: list[Mapping[str, Mapping[str, Any]]]
11
+ ) -> None:
12
+ """Register a list of HTTP request-response pairs."""
13
+ for call in http_calls:
14
+ request, response = call["request"], call["response"]
15
+ matcher = re.compile(request["url"]) if request["is_regex"] else request["url"]
16
+ mocker.register_uri(request["method"], matcher, **response)
@@ -0,0 +1,59 @@
1
+ # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
+
3
+
4
+ import importlib.util
5
+ from pathlib import Path
6
+ from types import ModuleType
7
+
8
+ import pytest
9
+
10
+ # The following fixtures are used to load a manifest-only connector's components module and manifest file.
11
+ # They can be accessed from any test file in the connector's unit_tests directory by importing them as follows:
12
+
13
+ # from airbyte_cdk.test.utils.manifest_only_fixtures import components_module, connector_dir, manifest_path
14
+
15
+ # individual components can then be referenced as: components_module.<CustomComponentClass>
16
+
17
+
18
+ @pytest.fixture(scope="session")
19
+ def connector_dir(request: pytest.FixtureRequest) -> Path:
20
+ """Return the connector's root directory."""
21
+
22
+ current_dir = Path(request.config.invocation_params.dir)
23
+
24
+ # If the tests are run locally from the connector's unit_tests directory, return the parent (connector) directory
25
+ if current_dir.name == "unit_tests":
26
+ return current_dir.parent
27
+ # In CI, the tests are run from the connector directory itself
28
+ return current_dir
29
+
30
+
31
+ @pytest.fixture(scope="session")
32
+ def components_module(connector_dir: Path) -> ModuleType | None:
33
+ """Load and return the components module from the connector directory.
34
+
35
+ This assumes the components module is located at <connector_dir>/components.py.
36
+ """
37
+ components_path = connector_dir / "components.py"
38
+ if not components_path.exists():
39
+ return None
40
+
41
+ components_spec = importlib.util.spec_from_file_location("components", components_path)
42
+ if components_spec is None:
43
+ return None
44
+
45
+ components_module = importlib.util.module_from_spec(components_spec)
46
+ if components_spec.loader is None:
47
+ return None
48
+
49
+ components_spec.loader.exec_module(components_module)
50
+ return components_module
51
+
52
+
53
+ @pytest.fixture(scope="session")
54
+ def manifest_path(connector_dir: Path) -> Path:
55
+ """Return the path to the connector's manifest file."""
56
+ path = connector_dir / "manifest.yaml"
57
+ if not path.exists():
58
+ raise FileNotFoundError(f"Manifest file not found at {path}")
59
+ return path