airbyte-cdk 0.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (368) hide show
  1. airbyte_cdk/__init__.py +358 -0
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +236 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +104 -0
  7. airbyte_cdk/connector.py +123 -0
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/__init__.py +3 -0
  10. airbyte_cdk/connector_builder/connector_builder_handler.py +121 -0
  11. airbyte_cdk/connector_builder/main.py +107 -0
  12. airbyte_cdk/connector_builder/models.py +73 -0
  13. airbyte_cdk/connector_builder/test_reader/__init__.py +7 -0
  14. airbyte_cdk/connector_builder/test_reader/helpers.py +689 -0
  15. airbyte_cdk/connector_builder/test_reader/message_grouper.py +173 -0
  16. airbyte_cdk/connector_builder/test_reader/reader.py +441 -0
  17. airbyte_cdk/connector_builder/test_reader/types.py +83 -0
  18. airbyte_cdk/destinations/__init__.py +8 -0
  19. airbyte_cdk/destinations/destination.py +154 -0
  20. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  21. airbyte_cdk/destinations/vector_db_based/__init__.py +38 -0
  22. airbyte_cdk/destinations/vector_db_based/config.py +298 -0
  23. airbyte_cdk/destinations/vector_db_based/document_processor.py +223 -0
  24. airbyte_cdk/destinations/vector_db_based/embedder.py +303 -0
  25. airbyte_cdk/destinations/vector_db_based/indexer.py +78 -0
  26. airbyte_cdk/destinations/vector_db_based/test_utils.py +63 -0
  27. airbyte_cdk/destinations/vector_db_based/utils.py +35 -0
  28. airbyte_cdk/destinations/vector_db_based/writer.py +104 -0
  29. airbyte_cdk/entrypoint.py +414 -0
  30. airbyte_cdk/exception_handler.py +56 -0
  31. airbyte_cdk/logger.py +109 -0
  32. airbyte_cdk/models/__init__.py +72 -0
  33. airbyte_cdk/models/airbyte_protocol.py +88 -0
  34. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  35. airbyte_cdk/models/well_known_types.py +5 -0
  36. airbyte_cdk/py.typed +0 -0
  37. airbyte_cdk/sources/__init__.py +26 -0
  38. airbyte_cdk/sources/abstract_source.py +326 -0
  39. airbyte_cdk/sources/concurrent_source/__init__.py +8 -0
  40. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +255 -0
  41. airbyte_cdk/sources/concurrent_source/concurrent_source.py +165 -0
  42. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +147 -0
  43. airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py +24 -0
  44. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  45. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +115 -0
  46. airbyte_cdk/sources/config.py +27 -0
  47. airbyte_cdk/sources/connector_state_manager.py +161 -0
  48. airbyte_cdk/sources/declarative/__init__.py +3 -0
  49. airbyte_cdk/sources/declarative/async_job/__init__.py +0 -0
  50. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  51. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +525 -0
  52. airbyte_cdk/sources/declarative/async_job/job_tracker.py +79 -0
  53. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  54. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  55. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  56. airbyte_cdk/sources/declarative/auth/__init__.py +8 -0
  57. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +42 -0
  58. airbyte_cdk/sources/declarative/auth/jwt.py +197 -0
  59. airbyte_cdk/sources/declarative/auth/oauth.py +293 -0
  60. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +45 -0
  61. airbyte_cdk/sources/declarative/auth/token.py +267 -0
  62. airbyte_cdk/sources/declarative/auth/token_provider.py +82 -0
  63. airbyte_cdk/sources/declarative/checks/__init__.py +24 -0
  64. airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +61 -0
  65. airbyte_cdk/sources/declarative/checks/check_stream.py +56 -0
  66. airbyte_cdk/sources/declarative/checks/connection_checker.py +35 -0
  67. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  68. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  69. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +526 -0
  70. airbyte_cdk/sources/declarative/datetime/__init__.py +3 -0
  71. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +65 -0
  72. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +118 -0
  73. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +3975 -0
  74. airbyte_cdk/sources/declarative/declarative_source.py +36 -0
  75. airbyte_cdk/sources/declarative/declarative_stream.py +241 -0
  76. airbyte_cdk/sources/declarative/decoders/__init__.py +33 -0
  77. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +218 -0
  78. airbyte_cdk/sources/declarative/decoders/decoder.py +32 -0
  79. airbyte_cdk/sources/declarative/decoders/decoder_parser.py +30 -0
  80. airbyte_cdk/sources/declarative/decoders/json_decoder.py +65 -0
  81. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  82. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  83. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  84. airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py +56 -0
  85. airbyte_cdk/sources/declarative/exceptions.py +9 -0
  86. airbyte_cdk/sources/declarative/extractors/__init__.py +21 -0
  87. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +86 -0
  88. airbyte_cdk/sources/declarative/extractors/http_selector.py +37 -0
  89. airbyte_cdk/sources/declarative/extractors/record_extractor.py +27 -0
  90. airbyte_cdk/sources/declarative/extractors/record_filter.py +91 -0
  91. airbyte_cdk/sources/declarative/extractors/record_selector.py +170 -0
  92. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +176 -0
  93. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  94. airbyte_cdk/sources/declarative/incremental/__init__.py +37 -0
  95. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +497 -0
  96. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +459 -0
  97. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  98. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +357 -0
  99. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +380 -0
  100. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  101. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  102. airbyte_cdk/sources/declarative/interpolation/__init__.py +9 -0
  103. airbyte_cdk/sources/declarative/interpolation/filters.py +139 -0
  104. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +66 -0
  105. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +56 -0
  106. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +52 -0
  107. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +79 -0
  108. airbyte_cdk/sources/declarative/interpolation/interpolation.py +34 -0
  109. airbyte_cdk/sources/declarative/interpolation/jinja.py +161 -0
  110. airbyte_cdk/sources/declarative/interpolation/macros.py +191 -0
  111. airbyte_cdk/sources/declarative/manifest_declarative_source.py +421 -0
  112. airbyte_cdk/sources/declarative/migrations/__init__.py +0 -0
  113. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  114. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  115. airbyte_cdk/sources/declarative/models/__init__.py +2 -0
  116. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2503 -0
  117. airbyte_cdk/sources/declarative/parsers/__init__.py +3 -0
  118. airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +157 -0
  119. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +21 -0
  120. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +172 -0
  121. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +213 -0
  122. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +3407 -0
  123. airbyte_cdk/sources/declarative/partition_routers/__init__.py +29 -0
  124. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  125. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  126. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +121 -0
  127. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  128. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +63 -0
  129. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +437 -0
  130. airbyte_cdk/sources/declarative/requesters/README.md +56 -0
  131. airbyte_cdk/sources/declarative/requesters/__init__.py +9 -0
  132. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +25 -0
  133. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +23 -0
  134. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +45 -0
  135. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +45 -0
  136. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +41 -0
  137. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +70 -0
  138. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +77 -0
  139. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +17 -0
  140. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +101 -0
  141. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +147 -0
  142. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  143. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +17 -0
  144. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +179 -0
  145. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +350 -0
  146. airbyte_cdk/sources/declarative/requesters/http_requester.py +433 -0
  147. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +21 -0
  148. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +327 -0
  149. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +76 -0
  150. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +65 -0
  151. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +25 -0
  152. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +98 -0
  153. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +102 -0
  154. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +71 -0
  155. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +48 -0
  156. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +66 -0
  157. airbyte_cdk/sources/declarative/requesters/request_option.py +117 -0
  158. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +23 -0
  159. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +92 -0
  160. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  161. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +59 -0
  162. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +68 -0
  163. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +119 -0
  164. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +79 -0
  165. airbyte_cdk/sources/declarative/requesters/request_path.py +15 -0
  166. airbyte_cdk/sources/declarative/requesters/requester.py +144 -0
  167. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  168. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  169. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  170. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  171. airbyte_cdk/sources/declarative/retrievers/__init__.py +19 -0
  172. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +124 -0
  173. airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
  174. airbyte_cdk/sources/declarative/retrievers/retriever.py +54 -0
  175. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +702 -0
  176. airbyte_cdk/sources/declarative/schema/__init__.py +25 -0
  177. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +47 -0
  178. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +285 -0
  179. airbyte_cdk/sources/declarative/schema/inline_schema_loader.py +19 -0
  180. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +92 -0
  181. airbyte_cdk/sources/declarative/schema/schema_loader.py +17 -0
  182. airbyte_cdk/sources/declarative/spec/__init__.py +7 -0
  183. airbyte_cdk/sources/declarative/spec/spec.py +48 -0
  184. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +7 -0
  185. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +93 -0
  186. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +25 -0
  187. airbyte_cdk/sources/declarative/transformations/__init__.py +17 -0
  188. airbyte_cdk/sources/declarative/transformations/add_fields.py +146 -0
  189. airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py +61 -0
  190. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  191. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  192. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  193. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  194. airbyte_cdk/sources/declarative/transformations/remove_fields.py +75 -0
  195. airbyte_cdk/sources/declarative/transformations/transformation.py +37 -0
  196. airbyte_cdk/sources/declarative/types.py +25 -0
  197. airbyte_cdk/sources/declarative/yaml_declarative_source.py +67 -0
  198. airbyte_cdk/sources/file_based/README.md +152 -0
  199. airbyte_cdk/sources/file_based/__init__.py +24 -0
  200. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +11 -0
  201. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +73 -0
  202. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +149 -0
  203. airbyte_cdk/sources/file_based/config/__init__.py +0 -0
  204. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +153 -0
  205. airbyte_cdk/sources/file_based/config/avro_format.py +25 -0
  206. airbyte_cdk/sources/file_based/config/csv_format.py +210 -0
  207. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  208. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +99 -0
  209. airbyte_cdk/sources/file_based/config/jsonl_format.py +18 -0
  210. airbyte_cdk/sources/file_based/config/parquet_format.py +25 -0
  211. airbyte_cdk/sources/file_based/config/unstructured_format.py +102 -0
  212. airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py +81 -0
  213. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +8 -0
  214. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +21 -0
  215. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +33 -0
  216. airbyte_cdk/sources/file_based/exceptions.py +159 -0
  217. airbyte_cdk/sources/file_based/file_based_source.py +466 -0
  218. airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py +123 -0
  219. airbyte_cdk/sources/file_based/file_based_stream_reader.py +209 -0
  220. airbyte_cdk/sources/file_based/file_record_data.py +22 -0
  221. airbyte_cdk/sources/file_based/file_types/__init__.py +37 -0
  222. airbyte_cdk/sources/file_based/file_types/avro_parser.py +233 -0
  223. airbyte_cdk/sources/file_based/file_types/csv_parser.py +527 -0
  224. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  225. airbyte_cdk/sources/file_based/file_types/file_transfer.py +30 -0
  226. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +86 -0
  227. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +145 -0
  228. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +275 -0
  229. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +480 -0
  230. airbyte_cdk/sources/file_based/remote_file.py +18 -0
  231. airbyte_cdk/sources/file_based/schema_helpers.py +281 -0
  232. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +17 -0
  233. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +20 -0
  234. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +52 -0
  235. airbyte_cdk/sources/file_based/stream/__init__.py +13 -0
  236. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +197 -0
  237. airbyte_cdk/sources/file_based/stream/concurrent/__init__.py +0 -0
  238. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +343 -0
  239. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +9 -0
  240. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +59 -0
  241. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +313 -0
  242. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +83 -0
  243. airbyte_cdk/sources/file_based/stream/cursor/__init__.py +4 -0
  244. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +66 -0
  245. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +149 -0
  246. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +396 -0
  247. airbyte_cdk/sources/file_based/stream/identities_stream.py +49 -0
  248. airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +92 -0
  249. airbyte_cdk/sources/file_based/types.py +10 -0
  250. airbyte_cdk/sources/http_config.py +10 -0
  251. airbyte_cdk/sources/http_logger.py +55 -0
  252. airbyte_cdk/sources/message/__init__.py +19 -0
  253. airbyte_cdk/sources/message/repository.py +137 -0
  254. airbyte_cdk/sources/source.py +95 -0
  255. airbyte_cdk/sources/specs/transfer_modes.py +26 -0
  256. airbyte_cdk/sources/streams/__init__.py +8 -0
  257. airbyte_cdk/sources/streams/availability_strategy.py +84 -0
  258. airbyte_cdk/sources/streams/call_rate.py +704 -0
  259. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  260. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  261. airbyte_cdk/sources/streams/checkpoint/cursor.py +77 -0
  262. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  263. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  264. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  265. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  266. airbyte_cdk/sources/streams/concurrent/__init__.py +3 -0
  267. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +96 -0
  268. airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py +37 -0
  269. airbyte_cdk/sources/streams/concurrent/adapters.py +397 -0
  270. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +94 -0
  271. airbyte_cdk/sources/streams/concurrent/clamping.py +99 -0
  272. airbyte_cdk/sources/streams/concurrent/cursor.py +481 -0
  273. airbyte_cdk/sources/streams/concurrent/cursor_types.py +32 -0
  274. airbyte_cdk/sources/streams/concurrent/default_stream.py +102 -0
  275. airbyte_cdk/sources/streams/concurrent/exceptions.py +18 -0
  276. airbyte_cdk/sources/streams/concurrent/helpers.py +42 -0
  277. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +64 -0
  278. airbyte_cdk/sources/streams/concurrent/partition_reader.py +45 -0
  279. airbyte_cdk/sources/streams/concurrent/partitions/__init__.py +3 -0
  280. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +48 -0
  281. airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py +18 -0
  282. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  283. airbyte_cdk/sources/streams/concurrent/partitions/types.py +38 -0
  284. airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py +0 -0
  285. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +182 -0
  286. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +223 -0
  287. airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
  288. airbyte_cdk/sources/streams/core.py +703 -0
  289. airbyte_cdk/sources/streams/http/__init__.py +10 -0
  290. airbyte_cdk/sources/streams/http/availability_strategy.py +54 -0
  291. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  292. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  293. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  294. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  295. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  296. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  297. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  298. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  299. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  300. airbyte_cdk/sources/streams/http/exceptions.py +61 -0
  301. airbyte_cdk/sources/streams/http/http.py +673 -0
  302. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  303. airbyte_cdk/sources/streams/http/rate_limiting.py +158 -0
  304. airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py +14 -0
  305. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +479 -0
  306. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +34 -0
  307. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +436 -0
  308. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +83 -0
  309. airbyte_cdk/sources/streams/permissions/identities_stream.py +75 -0
  310. airbyte_cdk/sources/streams/utils/__init__.py +3 -0
  311. airbyte_cdk/sources/types.py +169 -0
  312. airbyte_cdk/sources/utils/__init__.py +7 -0
  313. airbyte_cdk/sources/utils/casing.py +12 -0
  314. airbyte_cdk/sources/utils/files_directory.py +15 -0
  315. airbyte_cdk/sources/utils/record_helper.py +53 -0
  316. airbyte_cdk/sources/utils/schema_helpers.py +230 -0
  317. airbyte_cdk/sources/utils/slice_logger.py +57 -0
  318. airbyte_cdk/sources/utils/transform.py +277 -0
  319. airbyte_cdk/sources/utils/types.py +7 -0
  320. airbyte_cdk/sql/__init__.py +0 -0
  321. airbyte_cdk/sql/_util/__init__.py +0 -0
  322. airbyte_cdk/sql/_util/hashing.py +34 -0
  323. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  324. airbyte_cdk/sql/constants.py +32 -0
  325. airbyte_cdk/sql/exceptions.py +235 -0
  326. airbyte_cdk/sql/secrets.py +123 -0
  327. airbyte_cdk/sql/shared/__init__.py +15 -0
  328. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  329. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  330. airbyte_cdk/sql/types.py +160 -0
  331. airbyte_cdk/test/__init__.py +7 -0
  332. airbyte_cdk/test/catalog_builder.py +81 -0
  333. airbyte_cdk/test/entrypoint_wrapper.py +250 -0
  334. airbyte_cdk/test/mock_http/__init__.py +6 -0
  335. airbyte_cdk/test/mock_http/matcher.py +41 -0
  336. airbyte_cdk/test/mock_http/mocker.py +185 -0
  337. airbyte_cdk/test/mock_http/request.py +103 -0
  338. airbyte_cdk/test/mock_http/response.py +28 -0
  339. airbyte_cdk/test/mock_http/response_builder.py +237 -0
  340. airbyte_cdk/test/state_builder.py +33 -0
  341. airbyte_cdk/test/utils/__init__.py +1 -0
  342. airbyte_cdk/test/utils/data.py +24 -0
  343. airbyte_cdk/test/utils/http_mocking.py +16 -0
  344. airbyte_cdk/test/utils/manifest_only_fixtures.py +59 -0
  345. airbyte_cdk/test/utils/reading.py +26 -0
  346. airbyte_cdk/utils/__init__.py +10 -0
  347. airbyte_cdk/utils/airbyte_secrets_utils.py +80 -0
  348. airbyte_cdk/utils/analytics_message.py +25 -0
  349. airbyte_cdk/utils/constants.py +5 -0
  350. airbyte_cdk/utils/datetime_format_inferrer.py +94 -0
  351. airbyte_cdk/utils/datetime_helpers.py +499 -0
  352. airbyte_cdk/utils/event_timing.py +85 -0
  353. airbyte_cdk/utils/is_cloud_environment.py +18 -0
  354. airbyte_cdk/utils/mapping_helpers.py +162 -0
  355. airbyte_cdk/utils/message_utils.py +26 -0
  356. airbyte_cdk/utils/oneof_option_config.py +33 -0
  357. airbyte_cdk/utils/print_buffer.py +75 -0
  358. airbyte_cdk/utils/schema_inferrer.py +270 -0
  359. airbyte_cdk/utils/slice_hasher.py +37 -0
  360. airbyte_cdk/utils/spec_schema_transformations.py +26 -0
  361. airbyte_cdk/utils/stream_status_utils.py +43 -0
  362. airbyte_cdk/utils/traced_exception.py +145 -0
  363. airbyte_cdk-0.0.0.dev0.dist-info/LICENSE.txt +19 -0
  364. airbyte_cdk-0.0.0.dev0.dist-info/LICENSE_SHORT +1 -0
  365. airbyte_cdk-0.0.0.dev0.dist-info/METADATA +111 -0
  366. airbyte_cdk-0.0.0.dev0.dist-info/RECORD +368 -0
  367. airbyte_cdk-0.0.0.dev0.dist-info/WHEEL +4 -0
  368. airbyte_cdk-0.0.0.dev0.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,437 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+
6
+ import copy
7
+ import json
8
+ import logging
9
+ from dataclasses import InitVar, dataclass
10
+ from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
11
+
12
+ import dpath
13
+ import requests
14
+
15
+ from airbyte_cdk.models import AirbyteMessage
16
+ from airbyte_cdk.models import Type as MessageType
17
+ from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
18
+ from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
19
+ from airbyte_cdk.sources.declarative.requesters.request_option import (
20
+ RequestOption,
21
+ RequestOptionType,
22
+ )
23
+ from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
24
+ from airbyte_cdk.utils import AirbyteTracedException
25
+
26
+ if TYPE_CHECKING:
27
+ from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
28
+
29
+
30
+ @dataclass
31
+ class ParentStreamConfig:
32
+ """
33
+ Describes how to create a stream slice from a parent stream
34
+
35
+ stream: The stream to read records from
36
+ parent_key: The key of the parent stream's records that will be the stream slice key
37
+ partition_field: The partition key
38
+ extra_fields: Additional field paths to include in the stream slice
39
+ request_option: How to inject the slice value on an outgoing HTTP request
40
+ incremental_dependency (bool): Indicates if the parent stream should be read incrementally.
41
+ """
42
+
43
+ stream: "DeclarativeStream" # Parent streams must be DeclarativeStream because we can't know which part of the stream slice is a partition for regular Stream
44
+ parent_key: Union[InterpolatedString, str]
45
+ partition_field: Union[InterpolatedString, str]
46
+ config: Config
47
+ parameters: InitVar[Mapping[str, Any]]
48
+ extra_fields: Optional[Union[List[List[str]], List[List[InterpolatedString]]]] = (
49
+ None # List of field paths (arrays of strings)
50
+ )
51
+ request_option: Optional[RequestOption] = None
52
+ incremental_dependency: bool = False
53
+ lazy_read_pointer: Optional[List[Union[InterpolatedString, str]]] = None
54
+
55
+ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
56
+ self.parent_key = InterpolatedString.create(self.parent_key, parameters=parameters)
57
+ self.partition_field = InterpolatedString.create(
58
+ self.partition_field, parameters=parameters
59
+ )
60
+ if self.extra_fields:
61
+ # Create InterpolatedString for each field path in extra_keys
62
+ self.extra_fields = [
63
+ [InterpolatedString.create(path, parameters=parameters) for path in key_path]
64
+ for key_path in self.extra_fields
65
+ ]
66
+
67
+ self.lazy_read_pointer = (
68
+ [
69
+ InterpolatedString.create(path, parameters=parameters)
70
+ if isinstance(path, str)
71
+ else path
72
+ for path in self.lazy_read_pointer
73
+ ]
74
+ if self.lazy_read_pointer
75
+ else None
76
+ )
77
+
78
+
79
+ @dataclass
80
+ class SubstreamPartitionRouter(PartitionRouter):
81
+ """
82
+ Partition router that iterates over the parent's stream records and emits slices
83
+ Will populate the state with `partition_field` and `parent_slice` so they can be accessed by other components
84
+
85
+ Attributes:
86
+ parent_stream_configs (List[ParentStreamConfig]): parent streams to iterate over and their config
87
+ """
88
+
89
+ parent_stream_configs: List[ParentStreamConfig]
90
+ config: Config
91
+ parameters: InitVar[Mapping[str, Any]]
92
+
93
+ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
94
+ if not self.parent_stream_configs:
95
+ raise ValueError("SubstreamPartitionRouter needs at least 1 parent stream")
96
+ self._parameters = parameters
97
+
98
+ def get_request_params(
99
+ self,
100
+ stream_state: Optional[StreamState] = None,
101
+ stream_slice: Optional[StreamSlice] = None,
102
+ next_page_token: Optional[Mapping[str, Any]] = None,
103
+ ) -> Mapping[str, Any]:
104
+ # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response
105
+ return self._get_request_option(RequestOptionType.request_parameter, stream_slice)
106
+
107
+ def get_request_headers(
108
+ self,
109
+ stream_state: Optional[StreamState] = None,
110
+ stream_slice: Optional[StreamSlice] = None,
111
+ next_page_token: Optional[Mapping[str, Any]] = None,
112
+ ) -> Mapping[str, Any]:
113
+ # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response
114
+ return self._get_request_option(RequestOptionType.header, stream_slice)
115
+
116
+ def get_request_body_data(
117
+ self,
118
+ stream_state: Optional[StreamState] = None,
119
+ stream_slice: Optional[StreamSlice] = None,
120
+ next_page_token: Optional[Mapping[str, Any]] = None,
121
+ ) -> Mapping[str, Any]:
122
+ # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response
123
+ return self._get_request_option(RequestOptionType.body_data, stream_slice)
124
+
125
+ def get_request_body_json(
126
+ self,
127
+ stream_state: Optional[StreamState] = None,
128
+ stream_slice: Optional[StreamSlice] = None,
129
+ next_page_token: Optional[Mapping[str, Any]] = None,
130
+ ) -> Mapping[str, Any]:
131
+ # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response
132
+ return self._get_request_option(RequestOptionType.body_json, stream_slice)
133
+
134
+ def _get_request_option(
135
+ self, option_type: RequestOptionType, stream_slice: Optional[StreamSlice]
136
+ ) -> Mapping[str, Any]:
137
+ params: MutableMapping[str, Any] = {}
138
+ if stream_slice:
139
+ for parent_config in self.parent_stream_configs:
140
+ if (
141
+ parent_config.request_option
142
+ and parent_config.request_option.inject_into == option_type
143
+ ):
144
+ key = parent_config.partition_field.eval(self.config) # type: ignore # partition_field is always casted to an interpolated string
145
+ value = stream_slice.get(key)
146
+ if value:
147
+ parent_config.request_option.inject_into_request(params, value, self.config)
148
+ return params
149
+
150
+ def stream_slices(self) -> Iterable[StreamSlice]:
151
+ """
152
+ Iterate over each parent stream's record and create a StreamSlice for each record.
153
+
154
+ For each stream, iterate over its stream_slices.
155
+ For each stream slice, iterate over each record.
156
+ yield a stream slice for each such records.
157
+
158
+ If a parent slice contains no record, emit a slice with parent_record=None.
159
+
160
+ The template string can interpolate the following values:
161
+ - parent_stream_slice: mapping representing the parent's stream slice
162
+ - parent_record: mapping representing the parent record
163
+ - parent_stream_name: string representing the parent stream name
164
+ """
165
+ if not self.parent_stream_configs:
166
+ yield from []
167
+ else:
168
+ for parent_stream_config in self.parent_stream_configs:
169
+ parent_stream = parent_stream_config.stream
170
+ parent_field = parent_stream_config.parent_key.eval(self.config) # type: ignore # parent_key is always casted to an interpolated string
171
+ partition_field = parent_stream_config.partition_field.eval(self.config) # type: ignore # partition_field is always casted to an interpolated string
172
+ extra_fields = None
173
+ if parent_stream_config.extra_fields:
174
+ extra_fields = [
175
+ [field_path_part.eval(self.config) for field_path_part in field_path] # type: ignore [union-attr]
176
+ for field_path in parent_stream_config.extra_fields
177
+ ]
178
+
179
+ # read_stateless() assumes the parent is not concurrent. This is currently okay since the concurrent CDK does
180
+ # not support either substreams or RFR, but something that needs to be considered once we do
181
+ for parent_record in parent_stream.read_only_records():
182
+ parent_partition = None
183
+ # Skip non-records (eg AirbyteLogMessage)
184
+ if isinstance(parent_record, AirbyteMessage):
185
+ self.logger.warning(
186
+ f"Parent stream {parent_stream.name} returns records of type AirbyteMessage. This SubstreamPartitionRouter is not able to checkpoint incremental parent state."
187
+ )
188
+ if parent_record.type == MessageType.RECORD:
189
+ parent_record = parent_record.record.data # type: ignore[union-attr, assignment] # record is always a Record
190
+ else:
191
+ continue
192
+ elif isinstance(parent_record, Record):
193
+ parent_partition = (
194
+ parent_record.associated_slice.partition
195
+ if parent_record.associated_slice
196
+ else {}
197
+ )
198
+ parent_record = parent_record.data
199
+ elif not isinstance(parent_record, Mapping):
200
+ # The parent_record should only take the form of a Record, AirbyteMessage, or Mapping. Anything else is invalid
201
+ raise AirbyteTracedException(
202
+ message=f"Parent stream returned records as invalid type {type(parent_record)}"
203
+ )
204
+ try:
205
+ partition_value = dpath.get(
206
+ parent_record, # type: ignore [arg-type]
207
+ parent_field,
208
+ )
209
+ except KeyError:
210
+ continue
211
+
212
+ # Add extra fields
213
+ extracted_extra_fields = self._extract_extra_fields(parent_record, extra_fields)
214
+
215
+ if parent_stream_config.lazy_read_pointer:
216
+ extracted_extra_fields = {
217
+ "child_response": self._extract_child_response(
218
+ parent_record,
219
+ parent_stream_config.lazy_read_pointer, # type: ignore[arg-type] # lazy_read_pointer type handeled in __post_init__ of parent_stream_config
220
+ ),
221
+ **extracted_extra_fields,
222
+ }
223
+
224
+ yield StreamSlice(
225
+ partition={
226
+ partition_field: partition_value,
227
+ "parent_slice": parent_partition or {},
228
+ },
229
+ cursor_slice={},
230
+ extra_fields=extracted_extra_fields,
231
+ )
232
+
233
+ def _extract_child_response(
234
+ self, parent_record: Mapping[str, Any] | AirbyteMessage, pointer: List[InterpolatedString]
235
+ ) -> requests.Response:
236
+ """Extract child records from a parent record based on lazy pointers."""
237
+
238
+ def _create_response(data: MutableMapping[str, Any]) -> SafeResponse:
239
+ """Create a SafeResponse with the given data."""
240
+ response = SafeResponse()
241
+ response.content = json.dumps(data).encode("utf-8")
242
+ response.status_code = 200
243
+ return response
244
+
245
+ path = [path.eval(self.config) for path in pointer]
246
+ return _create_response(dpath.get(parent_record, path, default=[])) # type: ignore # argunet will be a MutableMapping, given input data structure
247
+
248
+ def _extract_extra_fields(
249
+ self,
250
+ parent_record: Mapping[str, Any] | AirbyteMessage,
251
+ extra_fields: Optional[List[List[str]]] = None,
252
+ ) -> Mapping[str, Any]:
253
+ """
254
+ Extracts additional fields specified by their paths from the parent record.
255
+
256
+ Args:
257
+ parent_record (Mapping[str, Any]): The record from the parent stream to extract fields from.
258
+ extra_fields (Optional[List[List[str]]]): A list of field paths (as lists of strings) to extract from the parent record.
259
+
260
+ Returns:
261
+ Mapping[str, Any]: A dictionary containing the extracted fields.
262
+ The keys are the joined field paths, and the values are the corresponding extracted values.
263
+ """
264
+ extracted_extra_fields = {}
265
+ if extra_fields:
266
+ for extra_field_path in extra_fields:
267
+ try:
268
+ extra_field_value = dpath.get(
269
+ parent_record, # type: ignore [arg-type]
270
+ extra_field_path,
271
+ )
272
+ self.logger.debug(
273
+ f"Extracted extra_field_path: {extra_field_path} with value: {extra_field_value}"
274
+ )
275
+ except KeyError:
276
+ self.logger.debug(f"Failed to extract extra_field_path: {extra_field_path}")
277
+ extra_field_value = None
278
+ extracted_extra_fields[".".join(extra_field_path)] = extra_field_value
279
+ return extracted_extra_fields
280
+
281
+ def set_initial_state(self, stream_state: StreamState) -> None:
282
+ """
283
+ Set the state of the parent streams.
284
+
285
+ If the `parent_state` key is missing from `stream_state`, migrate the child stream state to the parent stream's state format.
286
+ This migration applies only to parent streams with incremental dependencies.
287
+
288
+ Args:
289
+ stream_state (StreamState): The state of the streams to be set.
290
+
291
+ Example of state format:
292
+ {
293
+ "parent_state": {
294
+ "parent_stream_name1": {
295
+ "last_updated": "2023-05-27T00:00:00Z"
296
+ },
297
+ "parent_stream_name2": {
298
+ "last_updated": "2023-05-27T00:00:00Z"
299
+ }
300
+ }
301
+ }
302
+
303
+ Example of migrating to parent state format:
304
+ - Initial state:
305
+ {
306
+ "updated_at": "2023-05-27T00:00:00Z"
307
+ }
308
+ - After migration:
309
+ {
310
+ "updated_at": "2023-05-27T00:00:00Z",
311
+ "parent_state": {
312
+ "parent_stream_name": {
313
+ "parent_stream_cursor": "2023-05-27T00:00:00Z"
314
+ }
315
+ }
316
+ }
317
+ """
318
+ if not stream_state:
319
+ return
320
+
321
+ parent_state = stream_state.get("parent_state", {})
322
+
323
+ # Set state for each parent stream with an incremental dependency
324
+ for parent_config in self.parent_stream_configs:
325
+ if (
326
+ not parent_state.get(parent_config.stream.name, {})
327
+ and parent_config.incremental_dependency
328
+ ):
329
+ # Migrate child state to parent state format
330
+ parent_state = self._migrate_child_state_to_parent_state(stream_state)
331
+
332
+ if parent_config.incremental_dependency:
333
+ parent_config.stream.state = parent_state.get(parent_config.stream.name, {})
334
+
335
+ def _migrate_child_state_to_parent_state(self, stream_state: StreamState) -> StreamState:
336
+ """
337
+ Migrate the child or global stream state into the parent stream's state format.
338
+
339
+ This method converts the child stream state—or, if present, the global state—into a format that is
340
+ compatible with parent streams that use incremental synchronization. The migration occurs only for
341
+ parent streams with incremental dependencies. It filters out per-partition states and retains only the
342
+ global state in the form {cursor_field: cursor_value}.
343
+
344
+ The method supports multiple input formats:
345
+ - A simple global state, e.g.:
346
+ {"updated_at": "2023-05-27T00:00:00Z"}
347
+ - A state object that contains a "state" key (which is assumed to hold the global state), e.g.:
348
+ {"state": {"updated_at": "2023-05-27T00:00:00Z"}, ...}
349
+ In this case, the migration uses the first value from the "state" dictionary.
350
+ - Any per-partition state formats or other non-simple structures are ignored during migration.
351
+
352
+ Args:
353
+ stream_state (StreamState): The state to migrate. Expected formats include:
354
+ - {"updated_at": "2023-05-27T00:00:00Z"}
355
+ - {"state": {"updated_at": "2023-05-27T00:00:00Z"}, ...}
356
+ (In this format, only the first global state value is used, and per-partition states are ignored.)
357
+
358
+ Returns:
359
+ StreamState: A migrated state for parent streams in the format:
360
+ {
361
+ "parent_stream_name": {"parent_stream_cursor": "2023-05-27T00:00:00Z"}
362
+ }
363
+ where each parent stream with an incremental dependency is assigned its corresponding cursor value.
364
+
365
+ Example:
366
+ Input: {"updated_at": "2023-05-27T00:00:00Z"}
367
+ Output: {
368
+ "parent_stream_name": {"parent_stream_cursor": "2023-05-27T00:00:00Z"}
369
+ }
370
+ """
371
+ substream_state_values = list(stream_state.values())
372
+ substream_state = substream_state_values[0] if substream_state_values else {}
373
+
374
+ # Ignore per-partition states or invalid formats.
375
+ if isinstance(substream_state, (list, dict)) or len(substream_state_values) != 1:
376
+ # If a global state is present under the key "state", use its first value.
377
+ if "state" in stream_state and isinstance(stream_state["state"], dict):
378
+ substream_state = list(stream_state["state"].values())[0]
379
+ else:
380
+ return {}
381
+
382
+ # Build the parent state for all parent streams with incremental dependencies.
383
+ parent_state = {}
384
+ if substream_state:
385
+ for parent_config in self.parent_stream_configs:
386
+ if parent_config.incremental_dependency:
387
+ parent_state[parent_config.stream.name] = {
388
+ parent_config.stream.cursor_field: substream_state
389
+ }
390
+
391
+ return parent_state
392
+
393
+ def get_stream_state(self) -> Optional[Mapping[str, StreamState]]:
394
+ """
395
+ Get the state of the parent streams.
396
+
397
+ Returns:
398
+ StreamState: The current state of the parent streams.
399
+
400
+ Example of state format:
401
+ {
402
+ "parent_stream_name1": {
403
+ "last_updated": "2023-05-27T00:00:00Z"
404
+ },
405
+ "parent_stream_name2": {
406
+ "last_updated": "2023-05-27T00:00:00Z"
407
+ }
408
+ }
409
+ """
410
+ parent_state = {}
411
+ for parent_config in self.parent_stream_configs:
412
+ if parent_config.incremental_dependency:
413
+ parent_state[parent_config.stream.name] = copy.deepcopy(parent_config.stream.state)
414
+ return parent_state
415
+
416
+ @property
417
+ def logger(self) -> logging.Logger:
418
+ return logging.getLogger("airbyte.SubstreamPartitionRouter")
419
+
420
+
421
+ class SafeResponse(requests.Response):
422
+ """
423
+ A subclass of requests.Response that acts as an interface to migrate parsed child records
424
+ into a response object. This allows seamless interaction with child records as if they
425
+ were original response, ensuring compatibility with methods that expect requests.Response data type.
426
+ """
427
+
428
+ def __getattr__(self, name: str) -> Any:
429
+ return getattr(requests.Response, name, None)
430
+
431
+ @property
432
+ def content(self) -> Optional[bytes]:
433
+ return super().content
434
+
435
+ @content.setter
436
+ def content(self, value: Union[str, bytes]) -> None:
437
+ self._content = value.encode() if isinstance(value, str) else value
@@ -0,0 +1,56 @@
1
+ # AsyncHttpJobRepository sequence diagram
2
+
3
+ - Components marked as optional are not required and can be ignored.
4
+ - if `download_target_requester` is not provided, `download_target_extractor` will get urls from the `polling_response`
5
+ - interpolation_context, e.g. `creation_response` or `polling_response` can be obtained from stream_slice
6
+
7
+ ```mermaid
8
+ ---
9
+ title: AsyncHttpJobRepository Sequence Diagram
10
+ ---
11
+ sequenceDiagram
12
+ participant AsyncHttpJobRepository as AsyncOrchestrator
13
+ participant CreationRequester as creation_requester
14
+ participant PollingRequester as polling_requester
15
+ participant UrlRequester as download_target_requester (Optional)
16
+ participant DownloadRetriever as download_retriever
17
+ participant AbortRequester as abort_requester (Optional)
18
+ participant DeleteRequester as delete_requester (Optional)
19
+ participant Reporting Server as Async Reporting Server
20
+
21
+ AsyncHttpJobRepository ->> CreationRequester: Initiate job creation
22
+ CreationRequester ->> Reporting Server: Create job request
23
+ Reporting Server -->> CreationRequester: Job ID response
24
+ CreationRequester -->> AsyncHttpJobRepository: Job ID
25
+
26
+ loop Poll for job status
27
+ AsyncHttpJobRepository ->> PollingRequester: Check job status
28
+ PollingRequester ->> Reporting Server: Status request (interpolation_context: `creation_response`)
29
+ Reporting Server -->> PollingRequester: Status response
30
+ PollingRequester -->> AsyncHttpJobRepository: Job status
31
+ end
32
+
33
+ alt Status: Ready
34
+ AsyncHttpJobRepository ->> UrlRequester: Request download URLs (if applicable)
35
+ UrlRequester ->> Reporting Server: URL request (interpolation_context: `polling_response`)
36
+ Reporting Server -->> UrlRequester: Download URLs
37
+ UrlRequester -->> AsyncHttpJobRepository: Download URLs
38
+
39
+ AsyncHttpJobRepository ->> DownloadRetriever: Download reports
40
+ DownloadRetriever ->> Reporting Server: Retrieve report data (interpolation_context: `url`)
41
+ Reporting Server -->> DownloadRetriever: Report data
42
+ DownloadRetriever -->> AsyncHttpJobRepository: Report data
43
+ else Status: Failed
44
+ AsyncHttpJobRepository ->> AbortRequester: Send abort request
45
+ AbortRequester ->> Reporting Server: Abort job
46
+ Reporting Server -->> AbortRequester: Abort confirmation
47
+ AbortRequester -->> AsyncHttpJobRepository: Confirmation
48
+ end
49
+
50
+ AsyncHttpJobRepository ->> DeleteRequester: Send delete job request
51
+ DeleteRequester ->> Reporting Server: Delete job
52
+ Reporting Server -->> DeleteRequester: Deletion confirmation
53
+ DeleteRequester -->> AsyncHttpJobRepository: Confirmation
54
+
55
+
56
+ ```
@@ -0,0 +1,9 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester
6
+ from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption
7
+ from airbyte_cdk.sources.declarative.requesters.requester import Requester
8
+
9
+ __all__ = ["HttpRequester", "RequestOption", "Requester"]
@@ -0,0 +1,25 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import (
6
+ BackoffStrategy,
7
+ )
8
+ from airbyte_cdk.sources.declarative.requesters.error_handlers.composite_error_handler import (
9
+ CompositeErrorHandler,
10
+ )
11
+ from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import (
12
+ DefaultErrorHandler,
13
+ )
14
+ from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler
15
+ from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import (
16
+ HttpResponseFilter,
17
+ )
18
+
19
+ __all__ = [
20
+ "BackoffStrategy",
21
+ "CompositeErrorHandler",
22
+ "DefaultErrorHandler",
23
+ "ErrorHandler",
24
+ "HttpResponseFilter",
25
+ ]
@@ -0,0 +1,23 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.constant_backoff_strategy import (
6
+ ConstantBackoffStrategy,
7
+ )
8
+ from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import (
9
+ ExponentialBackoffStrategy,
10
+ )
11
+ from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.wait_time_from_header_backoff_strategy import (
12
+ WaitTimeFromHeaderBackoffStrategy,
13
+ )
14
+ from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.wait_until_time_from_header_backoff_strategy import (
15
+ WaitUntilTimeFromHeaderBackoffStrategy,
16
+ )
17
+
18
+ __all__ = [
19
+ "ConstantBackoffStrategy",
20
+ "ExponentialBackoffStrategy",
21
+ "WaitTimeFromHeaderBackoffStrategy",
22
+ "WaitUntilTimeFromHeaderBackoffStrategy",
23
+ ]
@@ -0,0 +1,45 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from dataclasses import InitVar, dataclass
6
+ from typing import Any, Mapping, Optional, Union
7
+
8
+ import requests
9
+
10
+ from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
11
+ from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
12
+ from airbyte_cdk.sources.types import Config
13
+
14
+
15
+ @dataclass
16
+ class ConstantBackoffStrategy(BackoffStrategy):
17
+ """
18
+ Backoff strategy with a constant backoff interval
19
+
20
+ Attributes:
21
+ backoff_time_in_seconds (float): time to backoff before retrying a retryable request.
22
+ """
23
+
24
+ backoff_time_in_seconds: Union[float, InterpolatedString, str]
25
+ parameters: InitVar[Mapping[str, Any]]
26
+ config: Config
27
+
28
+ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
29
+ if not isinstance(self.backoff_time_in_seconds, InterpolatedString):
30
+ self.backoff_time_in_seconds = str(self.backoff_time_in_seconds)
31
+ if isinstance(self.backoff_time_in_seconds, float):
32
+ self.backoff_time_in_seconds = InterpolatedString.create(
33
+ str(self.backoff_time_in_seconds), parameters=parameters
34
+ )
35
+ else:
36
+ self.backoff_time_in_seconds = InterpolatedString.create(
37
+ self.backoff_time_in_seconds, parameters=parameters
38
+ )
39
+
40
+ def backoff_time(
41
+ self,
42
+ response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
43
+ attempt_count: int,
44
+ ) -> Optional[float]:
45
+ return self.backoff_time_in_seconds.eval(self.config) # type: ignore # backoff_time_in_seconds is always cast to an interpolated string
@@ -0,0 +1,45 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from dataclasses import InitVar, dataclass
6
+ from typing import Any, Mapping, Optional, Union
7
+
8
+ import requests
9
+
10
+ from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
11
+ from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
12
+ from airbyte_cdk.sources.types import Config
13
+
14
+
15
+ @dataclass
16
+ class ExponentialBackoffStrategy(BackoffStrategy):
17
+ """
18
+ Backoff strategy with an exponential backoff interval
19
+
20
+ Attributes:
21
+ factor (float): multiplicative factor
22
+ """
23
+
24
+ parameters: InitVar[Mapping[str, Any]]
25
+ config: Config
26
+ factor: Union[float, InterpolatedString, str] = 5
27
+
28
+ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
29
+ if not isinstance(self.factor, InterpolatedString):
30
+ self.factor = str(self.factor)
31
+ if isinstance(self.factor, float):
32
+ self._factor = InterpolatedString.create(str(self.factor), parameters=parameters)
33
+ else:
34
+ self._factor = InterpolatedString.create(self.factor, parameters=parameters)
35
+
36
+ @property
37
+ def _retry_factor(self) -> float:
38
+ return self._factor.eval(self.config) # type: ignore # factor is always cast to an interpolated string
39
+
40
+ def backoff_time(
41
+ self,
42
+ response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
43
+ attempt_count: int,
44
+ ) -> Optional[float]:
45
+ return self._retry_factor * 2**attempt_count # type: ignore # factor is always cast to an interpolated string