airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (517) hide show
  1. airbyte_cdk/__init__.py +355 -6
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +29 -10
  7. airbyte_cdk/connector.py +24 -24
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
  10. airbyte_cdk/connector_builder/main.py +45 -13
  11. airbyte_cdk/connector_builder/message_grouper.py +189 -50
  12. airbyte_cdk/connector_builder/models.py +3 -2
  13. airbyte_cdk/destinations/__init__.py +4 -3
  14. airbyte_cdk/destinations/destination.py +54 -20
  15. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  16. airbyte_cdk/destinations/vector_db_based/config.py +40 -17
  17. airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
  18. airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
  19. airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
  20. airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
  21. airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
  22. airbyte_cdk/entrypoint.py +153 -44
  23. airbyte_cdk/exception_handler.py +21 -3
  24. airbyte_cdk/logger.py +30 -44
  25. airbyte_cdk/models/__init__.py +13 -2
  26. airbyte_cdk/models/airbyte_protocol.py +86 -1
  27. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  28. airbyte_cdk/models/file_transfer_record_message.py +13 -0
  29. airbyte_cdk/models/well_known_types.py +1 -1
  30. airbyte_cdk/sources/__init__.py +5 -1
  31. airbyte_cdk/sources/abstract_source.py +125 -79
  32. airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
  33. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
  34. airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
  35. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
  36. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  37. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
  38. airbyte_cdk/sources/config.py +3 -2
  39. airbyte_cdk/sources/connector_state_manager.py +49 -83
  40. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  41. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
  42. airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
  43. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  44. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  45. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  46. airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
  47. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
  48. airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
  49. airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
  50. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
  51. airbyte_cdk/sources/declarative/auth/token.py +28 -10
  52. airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
  53. airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
  54. airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
  55. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  56. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  57. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +421 -0
  58. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
  59. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
  60. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1213 -88
  61. airbyte_cdk/sources/declarative/declarative_source.py +5 -2
  62. airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
  63. airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
  64. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
  65. airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
  66. airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
  67. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  68. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  69. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  70. airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
  71. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
  72. airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
  73. airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
  74. airbyte_cdk/sources/declarative/extractors/record_filter.py +65 -8
  75. airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
  76. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
  77. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  78. airbyte_cdk/sources/declarative/incremental/__init__.py +25 -3
  79. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
  80. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  81. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
  82. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +159 -74
  83. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  84. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  85. airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
  86. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
  87. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
  88. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
  89. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
  90. airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
  91. airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
  92. airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
  93. airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
  94. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  95. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  96. airbyte_cdk/sources/declarative/models/__init__.py +1 -1
  97. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1329 -595
  98. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
  99. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
  100. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
  101. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1699 -226
  102. airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
  103. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  104. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  105. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
  106. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  107. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
  108. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
  109. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
  110. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
  111. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
  112. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
  113. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
  114. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
  115. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
  116. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
  117. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
  118. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
  119. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  120. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
  121. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
  122. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
  123. airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
  124. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
  125. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
  126. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
  127. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
  128. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
  129. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
  130. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
  131. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
  132. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
  133. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
  134. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
  135. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
  136. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  137. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
  138. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
  139. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
  140. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
  141. airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
  142. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  143. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  144. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  145. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  146. airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
  147. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
  148. airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
  149. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +228 -72
  150. airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
  151. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
  152. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
  153. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
  154. airbyte_cdk/sources/declarative/spec/spec.py +12 -5
  155. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
  156. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
  157. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
  158. airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
  159. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  160. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  161. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  162. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  163. airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
  164. airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
  165. airbyte_cdk/sources/declarative/types.py +19 -110
  166. airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
  167. airbyte_cdk/sources/embedded/base_integration.py +16 -5
  168. airbyte_cdk/sources/embedded/catalog.py +16 -4
  169. airbyte_cdk/sources/embedded/runner.py +19 -3
  170. airbyte_cdk/sources/embedded/tools.py +5 -2
  171. airbyte_cdk/sources/file_based/README.md +152 -0
  172. airbyte_cdk/sources/file_based/__init__.py +24 -0
  173. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
  174. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
  175. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
  176. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +58 -10
  177. airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
  178. airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
  179. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  180. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
  181. airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
  182. airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
  183. airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
  184. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
  185. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
  186. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
  187. airbyte_cdk/sources/file_based/exceptions.py +52 -15
  188. airbyte_cdk/sources/file_based/file_based_source.py +163 -33
  189. airbyte_cdk/sources/file_based/file_based_stream_reader.py +83 -5
  190. airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
  191. airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
  192. airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
  193. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  194. airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
  195. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
  196. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
  197. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
  198. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +145 -41
  199. airbyte_cdk/sources/file_based/remote_file.py +1 -1
  200. airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
  201. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
  202. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
  203. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
  204. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
  205. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
  206. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
  207. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
  208. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
  209. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
  210. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
  211. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
  212. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +175 -45
  213. airbyte_cdk/sources/http_logger.py +8 -3
  214. airbyte_cdk/sources/message/__init__.py +7 -1
  215. airbyte_cdk/sources/message/repository.py +18 -4
  216. airbyte_cdk/sources/source.py +42 -38
  217. airbyte_cdk/sources/streams/__init__.py +2 -2
  218. airbyte_cdk/sources/streams/availability_strategy.py +54 -3
  219. airbyte_cdk/sources/streams/call_rate.py +64 -21
  220. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  221. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  222. airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
  223. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  224. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  225. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  226. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  227. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
  228. airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
  229. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
  230. airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
  231. airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
  232. airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
  233. airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
  234. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
  235. airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
  236. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
  237. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  238. airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
  239. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
  240. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
  241. airbyte_cdk/sources/streams/core.py +412 -87
  242. airbyte_cdk/sources/streams/http/__init__.py +2 -1
  243. airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
  244. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  245. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  246. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  247. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  248. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  249. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  250. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  251. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  252. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  253. airbyte_cdk/sources/streams/http/exceptions.py +27 -7
  254. airbyte_cdk/sources/streams/http/http.py +369 -246
  255. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  256. airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
  257. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
  258. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
  259. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
  260. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
  261. airbyte_cdk/sources/types.py +154 -0
  262. airbyte_cdk/sources/utils/record_helper.py +36 -21
  263. airbyte_cdk/sources/utils/schema_helpers.py +13 -6
  264. airbyte_cdk/sources/utils/slice_logger.py +4 -1
  265. airbyte_cdk/sources/utils/transform.py +54 -20
  266. airbyte_cdk/sql/_util/hashing.py +34 -0
  267. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  268. airbyte_cdk/sql/constants.py +32 -0
  269. airbyte_cdk/sql/exceptions.py +235 -0
  270. airbyte_cdk/sql/secrets.py +123 -0
  271. airbyte_cdk/sql/shared/__init__.py +15 -0
  272. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  273. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  274. airbyte_cdk/sql/types.py +160 -0
  275. airbyte_cdk/test/catalog_builder.py +70 -18
  276. airbyte_cdk/test/entrypoint_wrapper.py +117 -42
  277. airbyte_cdk/test/mock_http/__init__.py +1 -1
  278. airbyte_cdk/test/mock_http/matcher.py +6 -0
  279. airbyte_cdk/test/mock_http/mocker.py +57 -10
  280. airbyte_cdk/test/mock_http/request.py +19 -3
  281. airbyte_cdk/test/mock_http/response.py +3 -1
  282. airbyte_cdk/test/mock_http/response_builder.py +32 -16
  283. airbyte_cdk/test/state_builder.py +18 -10
  284. airbyte_cdk/test/utils/__init__.py +1 -0
  285. airbyte_cdk/test/utils/data.py +24 -0
  286. airbyte_cdk/test/utils/http_mocking.py +16 -0
  287. airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
  288. airbyte_cdk/test/utils/reading.py +26 -0
  289. airbyte_cdk/utils/__init__.py +2 -1
  290. airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
  291. airbyte_cdk/utils/analytics_message.py +10 -2
  292. airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
  293. airbyte_cdk/utils/event_timing.py +10 -10
  294. airbyte_cdk/utils/mapping_helpers.py +3 -1
  295. airbyte_cdk/utils/message_utils.py +20 -11
  296. airbyte_cdk/utils/print_buffer.py +75 -0
  297. airbyte_cdk/utils/schema_inferrer.py +198 -28
  298. airbyte_cdk/utils/slice_hasher.py +30 -0
  299. airbyte_cdk/utils/spec_schema_transformations.py +6 -3
  300. airbyte_cdk/utils/stream_status_utils.py +8 -1
  301. airbyte_cdk/utils/traced_exception.py +61 -21
  302. airbyte_cdk-6.13.1.dev4106.dist-info/METADATA +109 -0
  303. airbyte_cdk-6.13.1.dev4106.dist-info/RECORD +349 -0
  304. {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.13.1.dev4106.dist-info}/WHEEL +1 -2
  305. airbyte_cdk-6.13.1.dev4106.dist-info/entry_points.txt +3 -0
  306. airbyte_cdk/sources/declarative/create_partial.py +0 -92
  307. airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
  308. airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
  309. airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
  310. airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
  311. airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
  312. airbyte_cdk/sources/deprecated/base_source.py +0 -94
  313. airbyte_cdk/sources/deprecated/client.py +0 -99
  314. airbyte_cdk/sources/singer/__init__.py +0 -8
  315. airbyte_cdk/sources/singer/singer_helpers.py +0 -304
  316. airbyte_cdk/sources/singer/source.py +0 -186
  317. airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
  318. airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
  319. airbyte_cdk/sources/streams/http/auth/core.py +0 -29
  320. airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
  321. airbyte_cdk/sources/streams/http/auth/token.py +0 -47
  322. airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
  323. airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
  324. airbyte_cdk/sources/utils/schema_models.py +0 -84
  325. airbyte_cdk-0.72.0.dist-info/METADATA +0 -243
  326. airbyte_cdk-0.72.0.dist-info/RECORD +0 -466
  327. airbyte_cdk-0.72.0.dist-info/top_level.txt +0 -3
  328. source_declarative_manifest/main.py +0 -29
  329. unit_tests/connector_builder/__init__.py +0 -3
  330. unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
  331. unit_tests/connector_builder/test_message_grouper.py +0 -713
  332. unit_tests/connector_builder/utils.py +0 -27
  333. unit_tests/destinations/test_destination.py +0 -243
  334. unit_tests/singer/test_singer_helpers.py +0 -56
  335. unit_tests/singer/test_singer_source.py +0 -112
  336. unit_tests/sources/__init__.py +0 -0
  337. unit_tests/sources/concurrent_source/__init__.py +0 -3
  338. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
  339. unit_tests/sources/declarative/__init__.py +0 -3
  340. unit_tests/sources/declarative/auth/__init__.py +0 -3
  341. unit_tests/sources/declarative/auth/test_oauth.py +0 -331
  342. unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
  343. unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
  344. unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
  345. unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
  346. unit_tests/sources/declarative/checks/__init__.py +0 -3
  347. unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
  348. unit_tests/sources/declarative/decoders/__init__.py +0 -0
  349. unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
  350. unit_tests/sources/declarative/external_component.py +0 -13
  351. unit_tests/sources/declarative/extractors/__init__.py +0 -3
  352. unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
  353. unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
  354. unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
  355. unit_tests/sources/declarative/incremental/__init__.py +0 -0
  356. unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
  357. unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
  358. unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
  359. unit_tests/sources/declarative/interpolation/__init__.py +0 -3
  360. unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
  361. unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
  362. unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
  363. unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
  364. unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
  365. unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
  366. unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
  367. unit_tests/sources/declarative/parsers/__init__.py +0 -3
  368. unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
  369. unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
  370. unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1841
  371. unit_tests/sources/declarative/parsers/testing_components.py +0 -36
  372. unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
  373. unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
  374. unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
  375. unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
  376. unit_tests/sources/declarative/requesters/__init__.py +0 -3
  377. unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
  378. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
  379. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
  380. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
  381. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
  382. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
  383. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
  384. unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
  385. unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
  386. unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
  387. unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
  388. unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
  389. unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
  390. unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
  391. unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
  392. unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
  393. unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
  394. unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
  395. unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
  396. unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
  397. unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
  398. unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
  399. unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
  400. unit_tests/sources/declarative/retrievers/__init__.py +0 -3
  401. unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
  402. unit_tests/sources/declarative/schema/__init__.py +0 -6
  403. unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
  404. unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
  405. unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
  406. unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
  407. unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
  408. unit_tests/sources/declarative/states/__init__.py +0 -3
  409. unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
  410. unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
  411. unit_tests/sources/declarative/test_create_partial.py +0 -83
  412. unit_tests/sources/declarative/test_declarative_stream.py +0 -103
  413. unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
  414. unit_tests/sources/declarative/test_types.py +0 -39
  415. unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
  416. unit_tests/sources/file_based/__init__.py +0 -0
  417. unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
  418. unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
  419. unit_tests/sources/file_based/config/__init__.py +0 -0
  420. unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
  421. unit_tests/sources/file_based/config/test_csv_format.py +0 -34
  422. unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
  423. unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
  424. unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
  425. unit_tests/sources/file_based/file_types/__init__.py +0 -0
  426. unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
  427. unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
  428. unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
  429. unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
  430. unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
  431. unit_tests/sources/file_based/helpers.py +0 -70
  432. unit_tests/sources/file_based/in_memory_files_source.py +0 -211
  433. unit_tests/sources/file_based/scenarios/__init__.py +0 -0
  434. unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
  435. unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
  436. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
  437. unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
  438. unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
  439. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
  440. unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
  441. unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
  442. unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
  443. unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
  444. unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
  445. unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
  446. unit_tests/sources/file_based/stream/__init__.py +0 -0
  447. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  448. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
  449. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
  450. unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
  451. unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
  452. unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
  453. unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
  454. unit_tests/sources/file_based/test_scenarios.py +0 -253
  455. unit_tests/sources/file_based/test_schema_helpers.py +0 -346
  456. unit_tests/sources/fixtures/__init__.py +0 -3
  457. unit_tests/sources/fixtures/source_test_fixture.py +0 -153
  458. unit_tests/sources/message/__init__.py +0 -0
  459. unit_tests/sources/message/test_repository.py +0 -153
  460. unit_tests/sources/streams/__init__.py +0 -0
  461. unit_tests/sources/streams/concurrent/__init__.py +0 -3
  462. unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
  463. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
  464. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
  465. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
  466. unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
  467. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
  468. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
  469. unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
  470. unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
  471. unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
  472. unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
  473. unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
  474. unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
  475. unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
  476. unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
  477. unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
  478. unit_tests/sources/streams/http/__init__.py +0 -0
  479. unit_tests/sources/streams/http/auth/__init__.py +0 -0
  480. unit_tests/sources/streams/http/auth/test_auth.py +0 -173
  481. unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
  482. unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
  483. unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
  484. unit_tests/sources/streams/http/test_http.py +0 -635
  485. unit_tests/sources/streams/test_availability_strategy.py +0 -70
  486. unit_tests/sources/streams/test_call_rate.py +0 -300
  487. unit_tests/sources/streams/test_stream_read.py +0 -405
  488. unit_tests/sources/streams/test_streams_core.py +0 -184
  489. unit_tests/sources/test_abstract_source.py +0 -1442
  490. unit_tests/sources/test_concurrent_source.py +0 -112
  491. unit_tests/sources/test_config.py +0 -92
  492. unit_tests/sources/test_connector_state_manager.py +0 -482
  493. unit_tests/sources/test_http_logger.py +0 -252
  494. unit_tests/sources/test_integration_source.py +0 -86
  495. unit_tests/sources/test_source.py +0 -684
  496. unit_tests/sources/test_source_read.py +0 -460
  497. unit_tests/test/__init__.py +0 -0
  498. unit_tests/test/mock_http/__init__.py +0 -0
  499. unit_tests/test/mock_http/test_matcher.py +0 -53
  500. unit_tests/test/mock_http/test_mocker.py +0 -214
  501. unit_tests/test/mock_http/test_request.py +0 -117
  502. unit_tests/test/mock_http/test_response_builder.py +0 -177
  503. unit_tests/test/test_entrypoint_wrapper.py +0 -240
  504. unit_tests/utils/__init__.py +0 -0
  505. unit_tests/utils/test_datetime_format_inferrer.py +0 -60
  506. unit_tests/utils/test_mapping_helpers.py +0 -54
  507. unit_tests/utils/test_message_utils.py +0 -91
  508. unit_tests/utils/test_rate_limiting.py +0 -26
  509. unit_tests/utils/test_schema_inferrer.py +0 -202
  510. unit_tests/utils/test_secret_utils.py +0 -135
  511. unit_tests/utils/test_stream_status_utils.py +0 -61
  512. unit_tests/utils/test_traced_exception.py +0 -107
  513. /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
  514. {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
  515. {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
  516. {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
  517. {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.13.1.dev4106.dist-info}/LICENSE.txt +0 -0
airbyte_cdk/__init__.py CHANGED
@@ -1,9 +1,358 @@
1
- #
2
1
  # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3
- #
2
+ """
3
+ # Welcome to the Airbyte Python CDK!
4
4
 
5
- from .connector import AirbyteSpec, Connector
6
- from .entrypoint import AirbyteEntrypoint
7
- from .logger import AirbyteLogger
5
+ The Airbyte Python CDK is a Python library that provides a set of tools to help you build
6
+ connectors for the Airbyte platform.
8
7
 
9
- __all__ = ["AirbyteEntrypoint", "AirbyteLogger", "AirbyteSpec", "Connector"]
8
+ ## Building Source Connectors
9
+
10
+ To build a source connector, you will want to refer to
11
+ the following classes and modules:
12
+
13
+ - `airbyte_cdk.sources`
14
+ - `airbyte_cdk.sources.concurrent_source`
15
+ - `airbyte_cdk.sources.config`
16
+ - `airbyte_cdk.sources.file_based`
17
+ - `airbyte_cdk.sources.streams`
18
+
19
+ ## Building Destination Connectors
20
+
21
+ To build a destination connector, you will want to refer to
22
+ the following classes and modules:
23
+
24
+ - `airbyte_cdk.destinations`
25
+ - `airbyte_cdk.destinations.Destination`
26
+ - `airbyte_cdk.destinations.vector_db_based`
27
+
28
+ ## Working with Airbyte Protocol Models
29
+
30
+ The Airbyte CDK provides a set of classes that help you work with the Airbyte protocol models:
31
+
32
+ - `airbyte_cdk.models.airbyte_protocol`
33
+ - `airbyte_cdk.models.airbyte_protocol_serializers`
34
+
35
+ ---
36
+
37
+ API Reference
38
+
39
+ ---
40
+
41
+ """
42
+
43
+ # Warning: The below imports are not stable and will cause circular
44
+ # dependencies if auto-sorted with isort. Please keep them in the same order.
45
+ # TODO: Submodules should import from lower-level modules, rather than importing from here.
46
+ # Imports should also be placed in `if TYPE_CHECKING` blocks if they are only used as type
47
+ # hints - again, to avoid circular dependencies.
48
+ # Once those issues are resolved, the below can be sorted with isort.
49
+ import dunamai as _dunamai
50
+
51
+ from .config_observation import (
52
+ create_connector_config_control_message,
53
+ emit_configuration_as_airbyte_control_message,
54
+ )
55
+ from .connector import BaseConnector, Connector
56
+ from .destinations import Destination
57
+ from .entrypoint import AirbyteEntrypoint, launch
58
+ from .logger import AirbyteLogFormatter, init_logger
59
+ from .models import (
60
+ AdvancedAuth,
61
+ AirbyteConnectionStatus,
62
+ AirbyteLogMessage,
63
+ AirbyteMessage,
64
+ AirbyteRecordMessage,
65
+ AirbyteStream,
66
+ ConfiguredAirbyteCatalog,
67
+ ConfiguredAirbyteStream,
68
+ ConnectorSpecification,
69
+ DestinationSyncMode,
70
+ FailureType,
71
+ Level,
72
+ OAuthConfigSpecification,
73
+ OrchestratorType,
74
+ Status,
75
+ SyncMode,
76
+ Type,
77
+ )
78
+ from .sources import AbstractSource, Source
79
+ from .sources.concurrent_source.concurrent_source import ConcurrentSource
80
+ from .sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
81
+ from .sources.config import BaseConfig
82
+ from .sources.connector_state_manager import ConnectorStateManager
83
+ from .sources.declarative.auth import DeclarativeOauth2Authenticator
84
+ from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth
85
+ from .sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator
86
+ from .sources.declarative.auth.token import (
87
+ ApiKeyAuthenticator,
88
+ BasicHttpAuthenticator,
89
+ BearerAuthenticator,
90
+ )
91
+ from .sources.declarative.datetime.min_max_datetime import MinMaxDatetime
92
+ from .sources.declarative.declarative_stream import DeclarativeStream
93
+ from .sources.declarative.decoders import Decoder, JsonDecoder
94
+ from .sources.declarative.exceptions import ReadException
95
+ from .sources.declarative.extractors import DpathExtractor, RecordSelector
96
+ from .sources.declarative.extractors.record_extractor import RecordExtractor
97
+ from .sources.declarative.extractors.record_filter import RecordFilter
98
+ from .sources.declarative.incremental import DatetimeBasedCursor
99
+ from .sources.declarative.interpolation import InterpolatedBoolean, InterpolatedString
100
+ from .sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
101
+ from .sources.declarative.migrations.legacy_to_per_partition_state_migration import (
102
+ LegacyToPerPartitionStateMigration,
103
+ )
104
+ from .sources.declarative.partition_routers import (
105
+ CartesianProductStreamSlicer,
106
+ SinglePartitionRouter,
107
+ SubstreamPartitionRouter,
108
+ )
109
+ from .sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig
110
+ from .sources.declarative.requesters import HttpRequester, Requester
111
+ from .sources.declarative.requesters.error_handlers import BackoffStrategy
112
+ from .sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy
113
+ from .sources.declarative.requesters.paginators.strategies import (
114
+ CursorPaginationStrategy,
115
+ OffsetIncrement,
116
+ PageIncrement,
117
+ StopConditionPaginationStrategyDecorator,
118
+ )
119
+ from .sources.declarative.requesters.request_option import RequestOption, RequestOptionType
120
+ from .sources.declarative.requesters.request_options.default_request_options_provider import (
121
+ DefaultRequestOptionsProvider,
122
+ )
123
+ from .sources.declarative.requesters.request_options.interpolated_request_input_provider import (
124
+ InterpolatedRequestInputProvider,
125
+ )
126
+ from .sources.declarative.requesters.requester import HttpMethod
127
+ from .sources.declarative.retrievers import SimpleRetriever
128
+ from .sources.declarative.schema import JsonFileSchemaLoader
129
+ from .sources.declarative.transformations.add_fields import AddedFieldDefinition, AddFields
130
+ from .sources.declarative.transformations.transformation import RecordTransformation
131
+ from .sources.declarative.types import FieldPointer
132
+ from .sources.declarative.yaml_declarative_source import YamlDeclarativeSource
133
+ from .sources.message import InMemoryMessageRepository, MessageRepository
134
+ from .sources.source import TState
135
+ from .sources.streams.availability_strategy import AvailabilityStrategy
136
+ from .sources.streams.call_rate import (
137
+ AbstractAPIBudget,
138
+ CachedLimiterSession,
139
+ HttpAPIBudget,
140
+ HttpRequestMatcher,
141
+ LimiterSession,
142
+ MovingWindowCallRatePolicy,
143
+ Rate,
144
+ )
145
+ from .sources.streams.checkpoint import Cursor as LegacyCursor
146
+ from .sources.streams.checkpoint import ResumableFullRefreshCursor
147
+ from .sources.streams.concurrent.adapters import StreamFacade
148
+ from .sources.streams.concurrent.cursor import (
149
+ ConcurrentCursor,
150
+ Cursor,
151
+ CursorField,
152
+ FinalStateCursor,
153
+ )
154
+ from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import (
155
+ EpochValueConcurrentStreamStateConverter,
156
+ IsoMillisConcurrentStreamStateConverter,
157
+ )
158
+ from .sources.streams.core import IncrementalMixin, Stream, package_name_from_class
159
+ from .sources.streams.http import HttpStream, HttpSubStream
160
+ from .sources.streams.http.availability_strategy import HttpAvailabilityStrategy
161
+ from .sources.streams.http.exceptions import (
162
+ BaseBackoffException,
163
+ DefaultBackoffException,
164
+ UserDefinedBackoffException,
165
+ )
166
+ from .sources.streams.http.rate_limiting import default_backoff_handler
167
+ from .sources.streams.http.requests_native_auth import (
168
+ Oauth2Authenticator,
169
+ SingleUseRefreshTokenOauth2Authenticator,
170
+ TokenAuthenticator,
171
+ )
172
+ from .sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
173
+ from .sources.types import Config, Record, StreamSlice
174
+ from .sources.utils import casing
175
+ from .sources.utils.schema_helpers import (
176
+ InternalConfig,
177
+ ResourceSchemaLoader,
178
+ check_config_against_spec_or_exit,
179
+ expand_refs,
180
+ split_config,
181
+ )
182
+ from .sources.utils.transform import TransformConfig, TypeTransformer
183
+ from .utils import AirbyteTracedException, is_cloud_environment
184
+ from .utils.constants import ENV_REQUEST_CACHE_PATH
185
+ from .utils.event_timing import create_timer
186
+ from .utils.oneof_option_config import OneOfOptionConfig
187
+ from .utils.spec_schema_transformations import resolve_refs
188
+ from .utils.stream_status_utils import as_airbyte_message
189
+
190
+ __all__ = [
191
+ # Availability strategy
192
+ "AvailabilityStrategy",
193
+ "HttpAvailabilityStrategy",
194
+ # Checkpoint
195
+ "LegacyCursor",
196
+ "ResumableFullRefreshCursor",
197
+ # Concurrent
198
+ "ConcurrentCursor",
199
+ "ConcurrentSource",
200
+ "ConcurrentSourceAdapter",
201
+ "Cursor",
202
+ "CursorField",
203
+ "DEFAULT_CONCURRENCY",
204
+ "EpochValueConcurrentStreamStateConverter",
205
+ "FinalStateCursor",
206
+ "IsoMillisConcurrentStreamStateConverter",
207
+ "StreamFacade",
208
+ # Config observation
209
+ "create_connector_config_control_message",
210
+ "emit_configuration_as_airbyte_control_message",
211
+ # Connector
212
+ "AbstractSource",
213
+ "BaseConfig",
214
+ "BaseConnector",
215
+ "Connector",
216
+ "Destination",
217
+ "Source",
218
+ "TState",
219
+ # Declarative
220
+ "AddFields",
221
+ "AddedFieldDefinition",
222
+ "ApiKeyAuthenticator",
223
+ "BackoffStrategy",
224
+ "BasicHttpAuthenticator",
225
+ "BearerAuthenticator",
226
+ "CartesianProductStreamSlicer",
227
+ "CursorPaginationStrategy",
228
+ "DatetimeBasedCursor",
229
+ "DeclarativeAuthenticator",
230
+ "DeclarativeOauth2Authenticator",
231
+ "DeclarativeSingleUseRefreshTokenOauth2Authenticator",
232
+ "DeclarativeStream",
233
+ "Decoder",
234
+ "DefaultPaginator",
235
+ "DefaultRequestOptionsProvider",
236
+ "DpathExtractor",
237
+ "FieldPointer",
238
+ "HttpMethod",
239
+ "HttpRequester",
240
+ "InterpolatedBoolean",
241
+ "InterpolatedRequestInputProvider",
242
+ "InterpolatedString",
243
+ "JsonDecoder",
244
+ "JsonFileSchemaLoader",
245
+ "LegacyToPerPartitionStateMigration",
246
+ "ManifestDeclarativeSource",
247
+ "MinMaxDatetime",
248
+ "NoAuth",
249
+ "OffsetIncrement",
250
+ "PageIncrement",
251
+ "PaginationStrategy",
252
+ "ParentStreamConfig",
253
+ "ReadException",
254
+ "RecordExtractor",
255
+ "RecordFilter",
256
+ "RecordSelector",
257
+ "RecordTransformation",
258
+ "RequestOption",
259
+ "RequestOptionType",
260
+ "Requester",
261
+ "ResponseStatus",
262
+ "SimpleRetriever",
263
+ "SinglePartitionRouter",
264
+ "StopConditionPaginationStrategyDecorator",
265
+ "StreamSlice",
266
+ "SubstreamPartitionRouter",
267
+ "YamlDeclarativeSource",
268
+ # Entrypoint
269
+ "launch",
270
+ "AirbyteEntrypoint",
271
+ # HTTP
272
+ "AbstractAPIBudget",
273
+ "AbstractHeaderAuthenticator",
274
+ "BaseBackoffException",
275
+ "CachedLimiterSession",
276
+ "DefaultBackoffException",
277
+ "default_backoff_handler",
278
+ "HttpAPIBudget",
279
+ "HttpAuthenticator",
280
+ "HttpRequestMatcher",
281
+ "HttpStream",
282
+ "HttpSubStream",
283
+ "LimiterSession",
284
+ "MovingWindowCallRatePolicy",
285
+ "MultipleTokenAuthenticator",
286
+ "Oauth2Authenticator",
287
+ "Rate",
288
+ "SingleUseRefreshTokenOauth2Authenticator",
289
+ "TokenAuthenticator",
290
+ "UserDefinedBackoffException",
291
+ # Logger
292
+ "AirbyteLogFormatter",
293
+ "init_logger",
294
+ # Protocol classes
295
+ "AirbyteStream",
296
+ "AirbyteConnectionStatus",
297
+ "AirbyteMessage",
298
+ "ConfiguredAirbyteCatalog",
299
+ "Status",
300
+ "Type",
301
+ "OrchestratorType",
302
+ "ConfiguredAirbyteStream",
303
+ "DestinationSyncMode",
304
+ "SyncMode",
305
+ "FailureType",
306
+ "AdvancedAuth",
307
+ "AirbyteLogMessage",
308
+ "OAuthConfigSpecification",
309
+ "ConnectorSpecification",
310
+ "Level",
311
+ "AirbyteRecordMessage",
312
+ # Repository
313
+ "InMemoryMessageRepository",
314
+ "MessageRepository",
315
+ # State management
316
+ "ConnectorStateManager",
317
+ # Stream
318
+ "IncrementalMixin",
319
+ "Stream",
320
+ "StreamData",
321
+ "package_name_from_class",
322
+ # Utils
323
+ "AirbyteTracedException",
324
+ "is_cloud_environment",
325
+ "casing",
326
+ "InternalConfig",
327
+ "ResourceSchemaLoader",
328
+ "check_config_against_spec_or_exit",
329
+ "split_config",
330
+ "TransformConfig",
331
+ "TypeTransformer",
332
+ "ENV_REQUEST_CACHE_PATH",
333
+ "create_timer",
334
+ "OneOfOptionConfig",
335
+ "resolve_refs",
336
+ "as_airbyte_message",
337
+ # Types
338
+ "Config",
339
+ "Record",
340
+ "Source",
341
+ "StreamSlice",
342
+ ]
343
+
344
+ __version__: str
345
+ """Version generated by poetry dynamic versioning during publish.
346
+
347
+ When running in development, dunamai will calculate a new prerelease version
348
+ from existing git release tag info.
349
+ """
350
+
351
+ try:
352
+ __version__ = _dunamai.get_version(
353
+ "airbyte-cdk",
354
+ third_choice=_dunamai.Version.from_any_vcs,
355
+ fallback=_dunamai.Version("0.0.0+dev"),
356
+ ).serialize()
357
+ except:
358
+ __version__ = "0.0.0+dev"
@@ -0,0 +1 @@
1
+ # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
@@ -0,0 +1,5 @@
1
+ from airbyte_cdk.cli.source_declarative_manifest._run import run
2
+
3
+ __all__ = [
4
+ "run",
5
+ ]
@@ -0,0 +1,230 @@
1
+ # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
+ """Defines the `source-declarative-manifest` connector, which installs alongside CDK.
3
+
4
+ This file was originally imported from the dedicated connector directory, under the
5
+ `airbyte` monorepo.
6
+
7
+ Usage:
8
+
9
+ ```
10
+ pipx install airbyte-cdk
11
+ source-declarative-manifest --help
12
+ source-declarative-manifest spec
13
+ ...
14
+ ```
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import json
20
+ import pkgutil
21
+ import sys
22
+ import traceback
23
+ from collections.abc import Mapping
24
+ from datetime import datetime
25
+ from pathlib import Path
26
+ from typing import Any, cast
27
+
28
+ import orjson
29
+
30
+ from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
31
+ from airbyte_cdk.models import (
32
+ AirbyteErrorTraceMessage,
33
+ AirbyteMessage,
34
+ AirbyteMessageSerializer,
35
+ AirbyteStateMessage,
36
+ AirbyteTraceMessage,
37
+ ConfiguredAirbyteCatalog,
38
+ ConnectorSpecificationSerializer,
39
+ TraceType,
40
+ Type,
41
+ )
42
+ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
43
+ ConcurrentDeclarativeSource,
44
+ )
45
+ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
46
+ from airbyte_cdk.sources.source import TState
47
+
48
+
49
+ class SourceLocalYaml(YamlDeclarativeSource):
50
+ """
51
+ Declarative source defined by a yaml file in the local filesystem
52
+ """
53
+
54
+ def __init__(
55
+ self,
56
+ catalog: ConfiguredAirbyteCatalog | None,
57
+ config: Mapping[str, Any] | None,
58
+ state: TState,
59
+ **kwargs: Any,
60
+ ) -> None:
61
+ """
62
+ HACK!
63
+ Problem: YamlDeclarativeSource relies on the calling module name/path to find the yaml file.
64
+ Implication: If you call YamlDeclarativeSource directly it will look for the yaml file in the wrong place. (e.g. the airbyte-cdk package)
65
+ Solution: Subclass YamlDeclarativeSource from the same location as the manifest to load.
66
+
67
+ When can we remove this?
68
+ When the airbyte-cdk is updated to not rely on the calling module name/path to find the yaml file.
69
+ When all manifest connectors are updated to use the new airbyte-cdk.
70
+ When all manifest connectors are updated to use the source-declarative-manifest as the base image.
71
+ """
72
+ super().__init__(
73
+ catalog=catalog,
74
+ config=config,
75
+ state=state, # type: ignore [arg-type]
76
+ path_to_yaml="manifest.yaml",
77
+ )
78
+
79
+
80
+ def _is_local_manifest_command(args: list[str]) -> bool:
81
+ # Check for a local manifest.yaml file
82
+ return Path("/airbyte/integration_code/source_declarative_manifest/manifest.yaml").exists()
83
+
84
+
85
+ def handle_command(args: list[str]) -> None:
86
+ if _is_local_manifest_command(args):
87
+ handle_local_manifest_command(args)
88
+ else:
89
+ handle_remote_manifest_command(args)
90
+
91
+
92
+ def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
93
+ try:
94
+ config, catalog, state = _parse_inputs_into_config_catalog_state(args)
95
+ return SourceLocalYaml(config=config, catalog=catalog, state=state)
96
+ except Exception as error:
97
+ print(
98
+ orjson.dumps(
99
+ AirbyteMessageSerializer.dump(
100
+ AirbyteMessage(
101
+ type=Type.TRACE,
102
+ trace=AirbyteTraceMessage(
103
+ type=TraceType.ERROR,
104
+ emitted_at=int(datetime.now().timestamp() * 1000),
105
+ error=AirbyteErrorTraceMessage(
106
+ message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
107
+ stack_trace=traceback.format_exc(),
108
+ ),
109
+ ),
110
+ )
111
+ )
112
+ ).decode()
113
+ )
114
+ raise error
115
+
116
+
117
+ def handle_local_manifest_command(args: list[str]) -> None:
118
+ source = _get_local_yaml_source(args)
119
+ launch(
120
+ source=source,
121
+ args=args,
122
+ )
123
+
124
+
125
+ def handle_remote_manifest_command(args: list[str]) -> None:
126
+ """Overrides the spec command to return the generalized spec for the declarative manifest source.
127
+
128
+ This is different from a typical low-code, but built and published separately source built as a ManifestDeclarativeSource,
129
+ because that will have a spec method that returns the spec for that specific source. Other than spec,
130
+ the generalized connector behaves the same as any other, since the manifest is provided in the config.
131
+ """
132
+ if args[0] == "spec":
133
+ json_spec = pkgutil.get_data(
134
+ "airbyte_cdk.cli.source_declarative_manifest",
135
+ "spec.json",
136
+ )
137
+ if json_spec is None:
138
+ raise FileNotFoundError(
139
+ "Could not find `spec.json` file for source-declarative-manifest"
140
+ )
141
+
142
+ spec_obj = json.loads(json_spec)
143
+ spec = ConnectorSpecificationSerializer.load(spec_obj)
144
+
145
+ message = AirbyteMessage(type=Type.SPEC, spec=spec)
146
+ print(AirbyteEntrypoint.airbyte_message_to_string(message))
147
+ else:
148
+ source = create_declarative_source(args)
149
+ launch(
150
+ source=source,
151
+ args=args,
152
+ )
153
+
154
+
155
+ def create_declarative_source(
156
+ args: list[str],
157
+ ) -> ConcurrentDeclarativeSource: # type: ignore [type-arg]
158
+ """Creates the source with the injected config.
159
+
160
+ This essentially does what other low-code sources do at build time, but at runtime,
161
+ with a user-provided manifest in the config. This better reflects what happens in the
162
+ connector builder.
163
+ """
164
+ try:
165
+ config: Mapping[str, Any] | None
166
+ catalog: ConfiguredAirbyteCatalog | None
167
+ state: list[AirbyteStateMessage]
168
+ config, catalog, state = _parse_inputs_into_config_catalog_state(args)
169
+ if config is None or "__injected_declarative_manifest" not in config:
170
+ raise ValueError(
171
+ "Invalid config: `__injected_declarative_manifest` should be provided at the root "
172
+ f"of the config but config only has keys: {list(config.keys() if config else [])}"
173
+ )
174
+ return ConcurrentDeclarativeSource(
175
+ config=config,
176
+ catalog=catalog,
177
+ state=state,
178
+ source_config=cast(dict[str, Any], config["__injected_declarative_manifest"]),
179
+ )
180
+ except Exception as error:
181
+ print(
182
+ orjson.dumps(
183
+ AirbyteMessageSerializer.dump(
184
+ AirbyteMessage(
185
+ type=Type.TRACE,
186
+ trace=AirbyteTraceMessage(
187
+ type=TraceType.ERROR,
188
+ emitted_at=int(datetime.now().timestamp() * 1000),
189
+ error=AirbyteErrorTraceMessage(
190
+ message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
191
+ stack_trace=traceback.format_exc(),
192
+ ),
193
+ ),
194
+ )
195
+ )
196
+ ).decode()
197
+ )
198
+ raise error
199
+
200
+
201
+ def _parse_inputs_into_config_catalog_state(
202
+ args: list[str],
203
+ ) -> tuple[
204
+ Mapping[str, Any] | None,
205
+ ConfiguredAirbyteCatalog | None,
206
+ list[AirbyteStateMessage],
207
+ ]:
208
+ parsed_args = AirbyteEntrypoint.parse_args(args)
209
+ config = (
210
+ ConcurrentDeclarativeSource.read_config(parsed_args.config)
211
+ if hasattr(parsed_args, "config")
212
+ else None
213
+ )
214
+ catalog = (
215
+ ConcurrentDeclarativeSource.read_catalog(parsed_args.catalog)
216
+ if hasattr(parsed_args, "catalog")
217
+ else None
218
+ )
219
+ state = (
220
+ ConcurrentDeclarativeSource.read_state(parsed_args.state)
221
+ if hasattr(parsed_args, "state")
222
+ else []
223
+ )
224
+
225
+ return config, catalog, state
226
+
227
+
228
+ def run() -> None:
229
+ args: list[str] = sys.argv[1:]
230
+ handle_command(args)
@@ -0,0 +1,17 @@
1
+ {
2
+ "documentationUrl": "https://docs.airbyte.com/integrations/sources/low-code",
3
+ "connectionSpecification": {
4
+ "$schema": "http://json-schema.org/draft-07/schema#",
5
+ "title": "Low-code source spec",
6
+ "type": "object",
7
+ "required": ["__injected_declarative_manifest"],
8
+ "additionalProperties": true,
9
+ "properties": {
10
+ "__injected_declarative_manifest": {
11
+ "title": "Low-code manifest",
12
+ "type": "object",
13
+ "description": "The low-code manifest that defines the components of the source."
14
+ }
15
+ }
16
+ }
17
+ }
@@ -7,14 +7,29 @@ from __future__ import ( # Used to evaluate type hints at runtime, a NameError:
7
7
  )
8
8
 
9
9
  import time
10
+ from copy import copy
10
11
  from typing import Any, List, MutableMapping
11
12
 
12
- from airbyte_cdk.models import AirbyteControlConnectorConfigMessage, AirbyteControlMessage, AirbyteMessage, OrchestratorType, Type
13
+ import orjson
14
+
15
+ from airbyte_cdk.models import (
16
+ AirbyteControlConnectorConfigMessage,
17
+ AirbyteControlMessage,
18
+ AirbyteMessage,
19
+ AirbyteMessageSerializer,
20
+ OrchestratorType,
21
+ Type,
22
+ )
13
23
 
14
24
 
15
- class ObservedDict(dict):
16
- def __init__(self, non_observed_mapping: MutableMapping, observer: ConfigObserver, update_on_unchanged_value=True) -> None:
17
- non_observed_mapping = non_observed_mapping.copy()
25
+ class ObservedDict(dict): # type: ignore # disallow_any_generics is set to True, and dict is equivalent to dict[Any]
26
+ def __init__(
27
+ self,
28
+ non_observed_mapping: MutableMapping[Any, Any],
29
+ observer: ConfigObserver,
30
+ update_on_unchanged_value: bool = True,
31
+ ) -> None:
32
+ non_observed_mapping = copy(non_observed_mapping)
18
33
  self.observer = observer
19
34
  self.update_on_unchanged_value = update_on_unchanged_value
20
35
  for item, value in non_observed_mapping.items():
@@ -29,7 +44,7 @@ class ObservedDict(dict):
29
44
  value[i] = ObservedDict(sub_value, observer)
30
45
  super().__init__(non_observed_mapping)
31
46
 
32
- def __setitem__(self, item: Any, value: Any):
47
+ def __setitem__(self, item: Any, value: Any) -> None:
33
48
  """Override dict.__setitem__ by:
34
49
  1. Observing the new value if it is a dict
35
50
  2. Call observer update if the new value is different from the previous one
@@ -58,25 +73,29 @@ class ConfigObserver:
58
73
  emit_configuration_as_airbyte_control_message(self.config)
59
74
 
60
75
 
61
- def observe_connector_config(non_observed_connector_config: MutableMapping[str, Any]):
76
+ def observe_connector_config(
77
+ non_observed_connector_config: MutableMapping[str, Any],
78
+ ) -> ObservedDict:
62
79
  if isinstance(non_observed_connector_config, ObservedDict):
63
80
  raise ValueError("This connector configuration is already observed")
64
81
  connector_config_observer = ConfigObserver()
65
- observed_connector_config = ObservedDict(non_observed_connector_config, connector_config_observer)
82
+ observed_connector_config = ObservedDict(
83
+ non_observed_connector_config, connector_config_observer
84
+ )
66
85
  connector_config_observer.set_config(observed_connector_config)
67
86
  return observed_connector_config
68
87
 
69
88
 
70
- def emit_configuration_as_airbyte_control_message(config: MutableMapping):
89
+ def emit_configuration_as_airbyte_control_message(config: MutableMapping[str, Any]) -> None:
71
90
  """
72
91
  WARNING: deprecated - emit_configuration_as_airbyte_control_message is being deprecated in favor of the MessageRepository mechanism.
73
92
  See the airbyte_cdk.sources.message package
74
93
  """
75
94
  airbyte_message = create_connector_config_control_message(config)
76
- print(airbyte_message.json(exclude_unset=True))
95
+ print(orjson.dumps(AirbyteMessageSerializer.dump(airbyte_message)).decode())
77
96
 
78
97
 
79
- def create_connector_config_control_message(config):
98
+ def create_connector_config_control_message(config: MutableMapping[str, Any]) -> AirbyteMessage:
80
99
  control_message = AirbyteControlMessage(
81
100
  type=OrchestratorType.CONNECTOR_CONFIG,
82
101
  emitted_at=time.time() * 1000,