airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev1__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (518) hide show
  1. airbyte_cdk/__init__.py +355 -6
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +29 -10
  7. airbyte_cdk/connector.py +24 -24
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
  10. airbyte_cdk/connector_builder/main.py +45 -13
  11. airbyte_cdk/connector_builder/message_grouper.py +189 -50
  12. airbyte_cdk/connector_builder/models.py +3 -2
  13. airbyte_cdk/destinations/__init__.py +4 -3
  14. airbyte_cdk/destinations/destination.py +54 -20
  15. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  16. airbyte_cdk/destinations/vector_db_based/config.py +40 -17
  17. airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
  18. airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
  19. airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
  20. airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
  21. airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
  22. airbyte_cdk/entrypoint.py +153 -44
  23. airbyte_cdk/exception_handler.py +21 -3
  24. airbyte_cdk/logger.py +30 -44
  25. airbyte_cdk/models/__init__.py +13 -2
  26. airbyte_cdk/models/airbyte_protocol.py +86 -1
  27. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  28. airbyte_cdk/models/file_transfer_record_message.py +13 -0
  29. airbyte_cdk/models/well_known_types.py +1 -1
  30. airbyte_cdk/sources/__init__.py +5 -1
  31. airbyte_cdk/sources/abstract_source.py +125 -79
  32. airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
  33. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
  34. airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
  35. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
  36. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  37. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
  38. airbyte_cdk/sources/config.py +3 -2
  39. airbyte_cdk/sources/connector_state_manager.py +49 -83
  40. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  41. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
  42. airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
  43. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  44. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  45. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  46. airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
  47. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
  48. airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
  49. airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
  50. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
  51. airbyte_cdk/sources/declarative/auth/token.py +28 -10
  52. airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
  53. airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
  54. airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
  55. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  56. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  57. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
  58. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
  59. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
  60. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1185 -85
  61. airbyte_cdk/sources/declarative/declarative_source.py +5 -2
  62. airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
  63. airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
  64. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
  65. airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
  66. airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
  67. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  68. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  69. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  70. airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
  71. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
  72. airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
  73. airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
  74. airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
  75. airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
  76. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
  77. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  78. airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
  79. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +340 -0
  80. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
  81. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  82. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
  83. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +174 -74
  84. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  85. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  86. airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
  87. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
  88. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
  89. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
  90. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
  91. airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
  92. airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
  93. airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
  94. airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
  95. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  96. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  97. airbyte_cdk/sources/declarative/models/__init__.py +1 -1
  98. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1319 -603
  99. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
  100. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
  101. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
  102. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1759 -225
  103. airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
  104. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  105. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  106. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
  107. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  108. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
  109. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
  110. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
  111. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
  112. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
  113. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
  114. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
  115. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
  116. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
  117. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
  118. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
  119. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
  120. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  121. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
  122. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
  123. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
  124. airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
  125. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
  126. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
  127. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
  128. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
  129. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
  130. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
  131. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
  132. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
  133. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
  134. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
  135. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
  136. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
  137. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  138. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
  139. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
  140. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
  141. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
  142. airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
  143. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  144. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  145. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  146. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  147. airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
  148. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
  149. airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
  150. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
  151. airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
  152. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
  153. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
  154. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
  155. airbyte_cdk/sources/declarative/spec/spec.py +12 -5
  156. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
  157. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
  158. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
  159. airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
  160. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  161. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  162. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  163. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  164. airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
  165. airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
  166. airbyte_cdk/sources/declarative/types.py +19 -110
  167. airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
  168. airbyte_cdk/sources/embedded/base_integration.py +16 -5
  169. airbyte_cdk/sources/embedded/catalog.py +16 -4
  170. airbyte_cdk/sources/embedded/runner.py +19 -3
  171. airbyte_cdk/sources/embedded/tools.py +5 -2
  172. airbyte_cdk/sources/file_based/README.md +152 -0
  173. airbyte_cdk/sources/file_based/__init__.py +24 -0
  174. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
  175. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
  176. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
  177. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
  178. airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
  179. airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
  180. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  181. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
  182. airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
  183. airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
  184. airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
  185. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
  186. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
  187. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
  188. airbyte_cdk/sources/file_based/exceptions.py +18 -15
  189. airbyte_cdk/sources/file_based/file_based_source.py +140 -33
  190. airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
  191. airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
  192. airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
  193. airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
  194. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  195. airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
  196. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
  197. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
  198. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
  199. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
  200. airbyte_cdk/sources/file_based/remote_file.py +1 -1
  201. airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
  202. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
  203. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
  204. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
  205. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
  206. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
  207. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
  208. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
  209. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
  210. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
  211. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
  212. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
  213. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
  214. airbyte_cdk/sources/http_logger.py +8 -3
  215. airbyte_cdk/sources/message/__init__.py +7 -1
  216. airbyte_cdk/sources/message/repository.py +18 -4
  217. airbyte_cdk/sources/source.py +42 -38
  218. airbyte_cdk/sources/streams/__init__.py +2 -2
  219. airbyte_cdk/sources/streams/availability_strategy.py +54 -3
  220. airbyte_cdk/sources/streams/call_rate.py +64 -21
  221. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  222. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  223. airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
  224. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  225. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  226. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  227. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  228. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
  229. airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
  230. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
  231. airbyte_cdk/sources/streams/concurrent/cursor.py +313 -48
  232. airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
  233. airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
  234. airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
  235. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
  236. airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
  237. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
  238. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  239. airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
  240. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
  241. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
  242. airbyte_cdk/sources/streams/core.py +412 -87
  243. airbyte_cdk/sources/streams/http/__init__.py +2 -1
  244. airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
  245. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  246. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  247. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  248. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  249. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  250. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  251. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  252. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  253. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  254. airbyte_cdk/sources/streams/http/exceptions.py +27 -7
  255. airbyte_cdk/sources/streams/http/http.py +369 -246
  256. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  257. airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
  258. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
  259. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
  260. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
  261. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
  262. airbyte_cdk/sources/types.py +154 -0
  263. airbyte_cdk/sources/utils/record_helper.py +36 -21
  264. airbyte_cdk/sources/utils/schema_helpers.py +13 -6
  265. airbyte_cdk/sources/utils/slice_logger.py +4 -1
  266. airbyte_cdk/sources/utils/transform.py +54 -20
  267. airbyte_cdk/sql/_util/hashing.py +34 -0
  268. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  269. airbyte_cdk/sql/constants.py +32 -0
  270. airbyte_cdk/sql/exceptions.py +235 -0
  271. airbyte_cdk/sql/secrets.py +123 -0
  272. airbyte_cdk/sql/shared/__init__.py +15 -0
  273. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  274. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  275. airbyte_cdk/sql/types.py +160 -0
  276. airbyte_cdk/test/catalog_builder.py +70 -18
  277. airbyte_cdk/test/entrypoint_wrapper.py +117 -42
  278. airbyte_cdk/test/mock_http/__init__.py +1 -1
  279. airbyte_cdk/test/mock_http/matcher.py +6 -0
  280. airbyte_cdk/test/mock_http/mocker.py +57 -10
  281. airbyte_cdk/test/mock_http/request.py +19 -3
  282. airbyte_cdk/test/mock_http/response.py +3 -1
  283. airbyte_cdk/test/mock_http/response_builder.py +32 -16
  284. airbyte_cdk/test/state_builder.py +18 -10
  285. airbyte_cdk/test/utils/__init__.py +1 -0
  286. airbyte_cdk/test/utils/data.py +24 -0
  287. airbyte_cdk/test/utils/http_mocking.py +16 -0
  288. airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
  289. airbyte_cdk/test/utils/reading.py +26 -0
  290. airbyte_cdk/utils/__init__.py +2 -1
  291. airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
  292. airbyte_cdk/utils/analytics_message.py +10 -2
  293. airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
  294. airbyte_cdk/utils/event_timing.py +10 -10
  295. airbyte_cdk/utils/mapping_helpers.py +3 -1
  296. airbyte_cdk/utils/message_utils.py +20 -11
  297. airbyte_cdk/utils/print_buffer.py +75 -0
  298. airbyte_cdk/utils/schema_inferrer.py +198 -28
  299. airbyte_cdk/utils/slice_hasher.py +30 -0
  300. airbyte_cdk/utils/spec_schema_transformations.py +6 -3
  301. airbyte_cdk/utils/stream_status_utils.py +8 -1
  302. airbyte_cdk/utils/traced_exception.py +61 -21
  303. airbyte_cdk-6.17.1.dev1.dist-info/METADATA +109 -0
  304. airbyte_cdk-6.17.1.dev1.dist-info/RECORD +350 -0
  305. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev1.dist-info}/WHEEL +1 -2
  306. airbyte_cdk-6.17.1.dev1.dist-info/entry_points.txt +3 -0
  307. airbyte_cdk/sources/declarative/create_partial.py +0 -92
  308. airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
  309. airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
  310. airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
  311. airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
  312. airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
  313. airbyte_cdk/sources/deprecated/base_source.py +0 -94
  314. airbyte_cdk/sources/deprecated/client.py +0 -99
  315. airbyte_cdk/sources/singer/__init__.py +0 -8
  316. airbyte_cdk/sources/singer/singer_helpers.py +0 -304
  317. airbyte_cdk/sources/singer/source.py +0 -186
  318. airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
  319. airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
  320. airbyte_cdk/sources/streams/http/auth/core.py +0 -29
  321. airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
  322. airbyte_cdk/sources/streams/http/auth/token.py +0 -47
  323. airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
  324. airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
  325. airbyte_cdk/sources/utils/schema_models.py +0 -84
  326. airbyte_cdk-0.72.1.dist-info/METADATA +0 -243
  327. airbyte_cdk-0.72.1.dist-info/RECORD +0 -466
  328. airbyte_cdk-0.72.1.dist-info/top_level.txt +0 -3
  329. source_declarative_manifest/main.py +0 -29
  330. unit_tests/connector_builder/__init__.py +0 -3
  331. unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
  332. unit_tests/connector_builder/test_message_grouper.py +0 -713
  333. unit_tests/connector_builder/utils.py +0 -27
  334. unit_tests/destinations/test_destination.py +0 -243
  335. unit_tests/singer/test_singer_helpers.py +0 -56
  336. unit_tests/singer/test_singer_source.py +0 -112
  337. unit_tests/sources/__init__.py +0 -0
  338. unit_tests/sources/concurrent_source/__init__.py +0 -3
  339. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
  340. unit_tests/sources/declarative/__init__.py +0 -3
  341. unit_tests/sources/declarative/auth/__init__.py +0 -3
  342. unit_tests/sources/declarative/auth/test_oauth.py +0 -331
  343. unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
  344. unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
  345. unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
  346. unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
  347. unit_tests/sources/declarative/checks/__init__.py +0 -3
  348. unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
  349. unit_tests/sources/declarative/decoders/__init__.py +0 -0
  350. unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
  351. unit_tests/sources/declarative/external_component.py +0 -13
  352. unit_tests/sources/declarative/extractors/__init__.py +0 -3
  353. unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
  354. unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
  355. unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
  356. unit_tests/sources/declarative/incremental/__init__.py +0 -0
  357. unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
  358. unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
  359. unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
  360. unit_tests/sources/declarative/interpolation/__init__.py +0 -3
  361. unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
  362. unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
  363. unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
  364. unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
  365. unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
  366. unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
  367. unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
  368. unit_tests/sources/declarative/parsers/__init__.py +0 -3
  369. unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
  370. unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
  371. unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1847
  372. unit_tests/sources/declarative/parsers/testing_components.py +0 -36
  373. unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
  374. unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
  375. unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
  376. unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
  377. unit_tests/sources/declarative/requesters/__init__.py +0 -3
  378. unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
  379. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
  380. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
  381. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
  382. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
  383. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
  384. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
  385. unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
  386. unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
  387. unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
  388. unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
  389. unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
  390. unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
  391. unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
  392. unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
  393. unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
  394. unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
  395. unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
  396. unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
  397. unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
  398. unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
  399. unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
  400. unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
  401. unit_tests/sources/declarative/retrievers/__init__.py +0 -3
  402. unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
  403. unit_tests/sources/declarative/schema/__init__.py +0 -6
  404. unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
  405. unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
  406. unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
  407. unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
  408. unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
  409. unit_tests/sources/declarative/states/__init__.py +0 -3
  410. unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
  411. unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
  412. unit_tests/sources/declarative/test_create_partial.py +0 -83
  413. unit_tests/sources/declarative/test_declarative_stream.py +0 -103
  414. unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
  415. unit_tests/sources/declarative/test_types.py +0 -39
  416. unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
  417. unit_tests/sources/file_based/__init__.py +0 -0
  418. unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
  419. unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
  420. unit_tests/sources/file_based/config/__init__.py +0 -0
  421. unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
  422. unit_tests/sources/file_based/config/test_csv_format.py +0 -34
  423. unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
  424. unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
  425. unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
  426. unit_tests/sources/file_based/file_types/__init__.py +0 -0
  427. unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
  428. unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
  429. unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
  430. unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
  431. unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
  432. unit_tests/sources/file_based/helpers.py +0 -70
  433. unit_tests/sources/file_based/in_memory_files_source.py +0 -211
  434. unit_tests/sources/file_based/scenarios/__init__.py +0 -0
  435. unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
  436. unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
  437. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
  438. unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
  439. unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
  440. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
  441. unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
  442. unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
  443. unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
  444. unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
  445. unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
  446. unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
  447. unit_tests/sources/file_based/stream/__init__.py +0 -0
  448. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  449. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
  450. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
  451. unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
  452. unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
  453. unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
  454. unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
  455. unit_tests/sources/file_based/test_scenarios.py +0 -253
  456. unit_tests/sources/file_based/test_schema_helpers.py +0 -346
  457. unit_tests/sources/fixtures/__init__.py +0 -3
  458. unit_tests/sources/fixtures/source_test_fixture.py +0 -153
  459. unit_tests/sources/message/__init__.py +0 -0
  460. unit_tests/sources/message/test_repository.py +0 -153
  461. unit_tests/sources/streams/__init__.py +0 -0
  462. unit_tests/sources/streams/concurrent/__init__.py +0 -3
  463. unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
  464. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
  465. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
  466. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
  467. unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
  468. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
  469. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
  470. unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
  471. unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
  472. unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
  473. unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
  474. unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
  475. unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
  476. unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
  477. unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
  478. unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
  479. unit_tests/sources/streams/http/__init__.py +0 -0
  480. unit_tests/sources/streams/http/auth/__init__.py +0 -0
  481. unit_tests/sources/streams/http/auth/test_auth.py +0 -173
  482. unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
  483. unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
  484. unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
  485. unit_tests/sources/streams/http/test_http.py +0 -635
  486. unit_tests/sources/streams/test_availability_strategy.py +0 -70
  487. unit_tests/sources/streams/test_call_rate.py +0 -300
  488. unit_tests/sources/streams/test_stream_read.py +0 -405
  489. unit_tests/sources/streams/test_streams_core.py +0 -184
  490. unit_tests/sources/test_abstract_source.py +0 -1442
  491. unit_tests/sources/test_concurrent_source.py +0 -112
  492. unit_tests/sources/test_config.py +0 -92
  493. unit_tests/sources/test_connector_state_manager.py +0 -482
  494. unit_tests/sources/test_http_logger.py +0 -252
  495. unit_tests/sources/test_integration_source.py +0 -86
  496. unit_tests/sources/test_source.py +0 -684
  497. unit_tests/sources/test_source_read.py +0 -460
  498. unit_tests/test/__init__.py +0 -0
  499. unit_tests/test/mock_http/__init__.py +0 -0
  500. unit_tests/test/mock_http/test_matcher.py +0 -53
  501. unit_tests/test/mock_http/test_mocker.py +0 -214
  502. unit_tests/test/mock_http/test_request.py +0 -117
  503. unit_tests/test/mock_http/test_response_builder.py +0 -177
  504. unit_tests/test/test_entrypoint_wrapper.py +0 -240
  505. unit_tests/utils/__init__.py +0 -0
  506. unit_tests/utils/test_datetime_format_inferrer.py +0 -60
  507. unit_tests/utils/test_mapping_helpers.py +0 -54
  508. unit_tests/utils/test_message_utils.py +0 -91
  509. unit_tests/utils/test_rate_limiting.py +0 -26
  510. unit_tests/utils/test_schema_inferrer.py +0 -202
  511. unit_tests/utils/test_secret_utils.py +0 -135
  512. unit_tests/utils/test_stream_status_utils.py +0 -61
  513. unit_tests/utils/test_traced_exception.py +0 -107
  514. /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
  515. {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
  516. {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
  517. {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
  518. {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev1.dist-info}/LICENSE.txt +0 -0
@@ -4,669 +4,1108 @@
4
4
  from __future__ import annotations
5
5
 
6
6
  from enum import Enum
7
- from typing import Any, Dict, List, Optional, Union
7
+ from typing import Any, Dict, List, Literal, Optional, Union
8
8
 
9
- from pydantic import BaseModel, Extra, Field
10
- from typing_extensions import Literal
9
+ from pydantic.v1 import BaseModel, Extra, Field
11
10
 
12
11
 
13
12
  class AuthFlowType(Enum):
14
- oauth2_0 = 'oauth2.0'
15
- oauth1_0 = 'oauth1.0'
13
+ oauth2_0 = "oauth2.0"
14
+ oauth1_0 = "oauth1.0"
16
15
 
17
16
 
18
17
  class BasicHttpAuthenticator(BaseModel):
19
- type: Literal['BasicHttpAuthenticator']
18
+ type: Literal["BasicHttpAuthenticator"]
20
19
  username: str = Field(
21
20
  ...,
22
- description='The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.',
21
+ description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.",
23
22
  examples=["{{ config['username'] }}", "{{ config['api_key'] }}"],
24
- title='Username',
23
+ title="Username",
25
24
  )
26
25
  password: Optional[str] = Field(
27
- '',
28
- description='The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.',
29
- examples=["{{ config['password'] }}", ''],
30
- title='Password',
26
+ "",
27
+ description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.",
28
+ examples=["{{ config['password'] }}", ""],
29
+ title="Password",
31
30
  )
32
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
31
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
33
32
 
34
33
 
35
34
  class BearerAuthenticator(BaseModel):
36
- type: Literal['BearerAuthenticator']
35
+ type: Literal["BearerAuthenticator"]
37
36
  api_token: str = Field(
38
37
  ...,
39
- description='Token to inject as request header for authenticating with the API.',
38
+ description="Token to inject as request header for authenticating with the API.",
40
39
  examples=["{{ config['api_key'] }}", "{{ config['token'] }}"],
41
- title='Bearer Token',
40
+ title="Bearer Token",
42
41
  )
43
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
42
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
44
43
 
45
44
 
46
45
  class CheckStream(BaseModel):
47
- type: Literal['CheckStream']
46
+ type: Literal["CheckStream"]
48
47
  stream_names: List[str] = Field(
49
48
  ...,
50
- description='Names of the streams to try reading from when running a check operation.',
51
- examples=[['users'], ['users', 'contacts']],
52
- title='Stream Names',
49
+ description="Names of the streams to try reading from when running a check operation.",
50
+ examples=[["users"], ["users", "contacts"]],
51
+ title="Stream Names",
53
52
  )
54
53
 
55
54
 
55
+ class ConcurrencyLevel(BaseModel):
56
+ type: Optional[Literal["ConcurrencyLevel"]] = None
57
+ default_concurrency: Union[int, str] = Field(
58
+ ...,
59
+ description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.",
60
+ examples=[10, "{{ config['num_workers'] or 10 }}"],
61
+ title="Default Concurrency",
62
+ )
63
+ max_concurrency: Optional[int] = Field(
64
+ None,
65
+ description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.",
66
+ examples=[20, 100],
67
+ title="Max Concurrency",
68
+ )
69
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
70
+
71
+
56
72
  class ConstantBackoffStrategy(BaseModel):
57
- type: Literal['ConstantBackoffStrategy']
73
+ type: Literal["ConstantBackoffStrategy"]
58
74
  backoff_time_in_seconds: Union[float, str] = Field(
59
75
  ...,
60
- description='Backoff time in seconds.',
76
+ description="Backoff time in seconds.",
61
77
  examples=[30, 30.5, "{{ config['backoff_time'] }}"],
62
- title='Backoff Time',
78
+ title="Backoff Time",
79
+ )
80
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
81
+
82
+
83
+ class CursorPagination(BaseModel):
84
+ type: Literal["CursorPagination"]
85
+ cursor_value: str = Field(
86
+ ...,
87
+ description="Value of the cursor defining the next page to fetch.",
88
+ examples=[
89
+ "{{ headers.link.next.cursor }}",
90
+ "{{ last_record['key'] }}",
91
+ "{{ response['nextPage'] }}",
92
+ ],
93
+ title="Cursor Value",
94
+ )
95
+ page_size: Optional[int] = Field(
96
+ None,
97
+ description="The number of records to include in each pages.",
98
+ examples=[100],
99
+ title="Page Size",
100
+ )
101
+ stop_condition: Optional[str] = Field(
102
+ None,
103
+ description="Template string evaluating when to stop paginating.",
104
+ examples=[
105
+ "{{ response.data.has_more is false }}",
106
+ "{{ 'next' not in headers['link'] }}",
107
+ ],
108
+ title="Stop Condition",
63
109
  )
64
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
110
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
65
111
 
66
112
 
67
113
  class CustomAuthenticator(BaseModel):
68
114
  class Config:
69
115
  extra = Extra.allow
70
116
 
71
- type: Literal['CustomAuthenticator']
117
+ type: Literal["CustomAuthenticator"]
72
118
  class_name: str = Field(
73
119
  ...,
74
- description='Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.',
75
- examples=['source_railz.components.ShortLivedTokenAuthenticator'],
76
- title='Class Name',
120
+ description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.",
121
+ examples=["source_railz.components.ShortLivedTokenAuthenticator"],
122
+ title="Class Name",
77
123
  )
78
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
124
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
79
125
 
80
126
 
81
127
  class CustomBackoffStrategy(BaseModel):
82
128
  class Config:
83
129
  extra = Extra.allow
84
130
 
85
- type: Literal['CustomBackoffStrategy']
131
+ type: Literal["CustomBackoffStrategy"]
86
132
  class_name: str = Field(
87
133
  ...,
88
- description='Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.',
89
- examples=['source_railz.components.MyCustomBackoffStrategy'],
90
- title='Class Name',
134
+ description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.",
135
+ examples=["source_railz.components.MyCustomBackoffStrategy"],
136
+ title="Class Name",
91
137
  )
92
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
138
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
93
139
 
94
140
 
95
141
  class CustomErrorHandler(BaseModel):
96
142
  class Config:
97
143
  extra = Extra.allow
98
144
 
99
- type: Literal['CustomErrorHandler']
145
+ type: Literal["CustomErrorHandler"]
100
146
  class_name: str = Field(
101
147
  ...,
102
- description='Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.',
103
- examples=['source_railz.components.MyCustomErrorHandler'],
104
- title='Class Name',
148
+ description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.",
149
+ examples=["source_railz.components.MyCustomErrorHandler"],
150
+ title="Class Name",
105
151
  )
106
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
152
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
107
153
 
108
154
 
109
155
  class CustomIncrementalSync(BaseModel):
110
156
  class Config:
111
157
  extra = Extra.allow
112
158
 
113
- type: Literal['CustomIncrementalSync']
159
+ type: Literal["CustomIncrementalSync"]
114
160
  class_name: str = Field(
115
161
  ...,
116
- description='Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.',
117
- examples=['source_railz.components.MyCustomIncrementalSync'],
118
- title='Class Name',
162
+ description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.",
163
+ examples=["source_railz.components.MyCustomIncrementalSync"],
164
+ title="Class Name",
119
165
  )
120
166
  cursor_field: str = Field(
121
167
  ...,
122
- description='The location of the value on a record that will be used as a bookmark during sync.',
168
+ description="The location of the value on a record that will be used as a bookmark during sync.",
123
169
  )
124
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
170
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
125
171
 
126
172
 
127
173
  class CustomPaginationStrategy(BaseModel):
128
174
  class Config:
129
175
  extra = Extra.allow
130
176
 
131
- type: Literal['CustomPaginationStrategy']
177
+ type: Literal["CustomPaginationStrategy"]
132
178
  class_name: str = Field(
133
179
  ...,
134
- description='Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.',
135
- examples=['source_railz.components.MyCustomPaginationStrategy'],
136
- title='Class Name',
180
+ description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.",
181
+ examples=["source_railz.components.MyCustomPaginationStrategy"],
182
+ title="Class Name",
137
183
  )
138
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
184
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
139
185
 
140
186
 
141
187
  class CustomRecordExtractor(BaseModel):
142
188
  class Config:
143
189
  extra = Extra.allow
144
190
 
145
- type: Literal['CustomRecordExtractor']
191
+ type: Literal["CustomRecordExtractor"]
146
192
  class_name: str = Field(
147
193
  ...,
148
- description='Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.',
149
- examples=['source_railz.components.MyCustomRecordExtractor'],
150
- title='Class Name',
194
+ description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.",
195
+ examples=["source_railz.components.MyCustomRecordExtractor"],
196
+ title="Class Name",
151
197
  )
152
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
198
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
153
199
 
154
200
 
155
201
  class CustomRecordFilter(BaseModel):
156
202
  class Config:
157
203
  extra = Extra.allow
158
204
 
159
- type: Literal['CustomRecordFilter']
205
+ type: Literal["CustomRecordFilter"]
160
206
  class_name: str = Field(
161
207
  ...,
162
- description='Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.',
163
- examples=['source_railz.components.MyCustomCustomRecordFilter'],
164
- title='Class Name',
208
+ description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.",
209
+ examples=["source_railz.components.MyCustomCustomRecordFilter"],
210
+ title="Class Name",
165
211
  )
166
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
212
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
167
213
 
168
214
 
169
215
  class CustomRequester(BaseModel):
170
216
  class Config:
171
217
  extra = Extra.allow
172
218
 
173
- type: Literal['CustomRequester']
219
+ type: Literal["CustomRequester"]
174
220
  class_name: str = Field(
175
221
  ...,
176
- description='Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.',
177
- examples=['source_railz.components.MyCustomRecordExtractor'],
178
- title='Class Name',
222
+ description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.",
223
+ examples=["source_railz.components.MyCustomRecordExtractor"],
224
+ title="Class Name",
179
225
  )
180
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
226
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
181
227
 
182
228
 
183
229
  class CustomRetriever(BaseModel):
184
230
  class Config:
185
231
  extra = Extra.allow
186
232
 
187
- type: Literal['CustomRetriever']
233
+ type: Literal["CustomRetriever"]
188
234
  class_name: str = Field(
189
235
  ...,
190
- description='Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.',
191
- examples=['source_railz.components.MyCustomRetriever'],
192
- title='Class Name',
236
+ description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.",
237
+ examples=["source_railz.components.MyCustomRetriever"],
238
+ title="Class Name",
193
239
  )
194
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
240
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
195
241
 
196
242
 
197
243
  class CustomPartitionRouter(BaseModel):
198
244
  class Config:
199
245
  extra = Extra.allow
200
246
 
201
- type: Literal['CustomPartitionRouter']
247
+ type: Literal["CustomPartitionRouter"]
202
248
  class_name: str = Field(
203
249
  ...,
204
- description='Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.',
205
- examples=['source_railz.components.MyCustomPartitionRouter'],
206
- title='Class Name',
250
+ description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.",
251
+ examples=["source_railz.components.MyCustomPartitionRouter"],
252
+ title="Class Name",
207
253
  )
208
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
254
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
209
255
 
210
256
 
211
257
  class CustomSchemaLoader(BaseModel):
212
258
  class Config:
213
259
  extra = Extra.allow
214
260
 
215
- type: Literal['CustomSchemaLoader']
261
+ type: Literal["CustomSchemaLoader"]
216
262
  class_name: str = Field(
217
263
  ...,
218
- description='Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.',
219
- examples=['source_railz.components.MyCustomSchemaLoader'],
220
- title='Class Name',
264
+ description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.",
265
+ examples=["source_railz.components.MyCustomSchemaLoader"],
266
+ title="Class Name",
221
267
  )
222
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
268
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
269
+
270
+
271
+ class CustomSchemaNormalization(BaseModel):
272
+ class Config:
273
+ extra = Extra.allow
274
+
275
+ type: Literal["CustomSchemaNormalization"]
276
+ class_name: str = Field(
277
+ ...,
278
+ description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.",
279
+ examples=[
280
+ "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer"
281
+ ],
282
+ title="Class Name",
283
+ )
284
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
285
+
286
+
287
+ class CustomStateMigration(BaseModel):
288
+ class Config:
289
+ extra = Extra.allow
290
+
291
+ type: Literal["CustomStateMigration"]
292
+ class_name: str = Field(
293
+ ...,
294
+ description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.",
295
+ examples=["source_railz.components.MyCustomStateMigration"],
296
+ title="Class Name",
297
+ )
298
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
223
299
 
224
300
 
225
301
  class CustomTransformation(BaseModel):
226
302
  class Config:
227
303
  extra = Extra.allow
228
304
 
229
- type: Literal['CustomTransformation']
305
+ type: Literal["CustomTransformation"]
230
306
  class_name: str = Field(
231
307
  ...,
232
- description='Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.',
233
- examples=['source_railz.components.MyCustomTransformation'],
234
- title='Class Name',
308
+ description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.",
309
+ examples=["source_railz.components.MyCustomTransformation"],
310
+ title="Class Name",
235
311
  )
236
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
312
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
313
+
314
+
315
+ class LegacyToPerPartitionStateMigration(BaseModel):
316
+ class Config:
317
+ extra = Extra.allow
318
+
319
+ type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None
320
+
321
+
322
+ class Algorithm(Enum):
323
+ HS256 = "HS256"
324
+ HS384 = "HS384"
325
+ HS512 = "HS512"
326
+ ES256 = "ES256"
327
+ ES256K = "ES256K"
328
+ ES384 = "ES384"
329
+ ES512 = "ES512"
330
+ RS256 = "RS256"
331
+ RS384 = "RS384"
332
+ RS512 = "RS512"
333
+ PS256 = "PS256"
334
+ PS384 = "PS384"
335
+ PS512 = "PS512"
336
+ EdDSA = "EdDSA"
337
+
338
+
339
+ class JwtHeaders(BaseModel):
340
+ class Config:
341
+ extra = Extra.forbid
342
+
343
+ kid: Optional[str] = Field(
344
+ None,
345
+ description="Private key ID for user account.",
346
+ examples=["{{ config['kid'] }}"],
347
+ title="Key Identifier",
348
+ )
349
+ typ: Optional[str] = Field(
350
+ "JWT",
351
+ description="The media type of the complete JWT.",
352
+ examples=["JWT"],
353
+ title="Type",
354
+ )
355
+ cty: Optional[str] = Field(
356
+ None,
357
+ description="Content type of JWT header.",
358
+ examples=["JWT"],
359
+ title="Content Type",
360
+ )
361
+
362
+
363
+ class JwtPayload(BaseModel):
364
+ class Config:
365
+ extra = Extra.forbid
366
+
367
+ iss: Optional[str] = Field(
368
+ None,
369
+ description="The user/principal that issued the JWT. Commonly a value unique to the user.",
370
+ examples=["{{ config['iss'] }}"],
371
+ title="Issuer",
372
+ )
373
+ sub: Optional[str] = Field(
374
+ None,
375
+ description="The subject of the JWT. Commonly defined by the API.",
376
+ title="Subject",
377
+ )
378
+ aud: Optional[str] = Field(
379
+ None,
380
+ description="The recipient that the JWT is intended for. Commonly defined by the API.",
381
+ examples=["appstoreconnect-v1"],
382
+ title="Audience",
383
+ )
384
+
385
+
386
+ class JwtAuthenticator(BaseModel):
387
+ type: Literal["JwtAuthenticator"]
388
+ secret_key: str = Field(
389
+ ...,
390
+ description="Secret used to sign the JSON web token.",
391
+ examples=["{{ config['secret_key'] }}"],
392
+ )
393
+ base64_encode_secret_key: Optional[bool] = Field(
394
+ False,
395
+ description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.',
396
+ )
397
+ algorithm: Algorithm = Field(
398
+ ...,
399
+ description="Algorithm used to sign the JSON web token.",
400
+ examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"],
401
+ )
402
+ token_duration: Optional[int] = Field(
403
+ 1200,
404
+ description="The amount of time in seconds a JWT token can be valid after being issued.",
405
+ examples=[1200, 3600],
406
+ title="Token Duration",
407
+ )
408
+ header_prefix: Optional[str] = Field(
409
+ None,
410
+ description="The prefix to be used within the Authentication header.",
411
+ examples=["Bearer", "Basic"],
412
+ title="Header Prefix",
413
+ )
414
+ jwt_headers: Optional[JwtHeaders] = Field(
415
+ None,
416
+ description="JWT headers used when signing JSON web token.",
417
+ title="JWT Headers",
418
+ )
419
+ additional_jwt_headers: Optional[Dict[str, Any]] = Field(
420
+ None,
421
+ description="Additional headers to be included with the JWT headers object.",
422
+ title="Additional JWT Headers",
423
+ )
424
+ jwt_payload: Optional[JwtPayload] = Field(
425
+ None,
426
+ description="JWT Payload used when signing JSON web token.",
427
+ title="JWT Payload",
428
+ )
429
+ additional_jwt_payload: Optional[Dict[str, Any]] = Field(
430
+ None,
431
+ description="Additional properties to be added to the JWT payload.",
432
+ title="Additional JWT Payload Properties",
433
+ )
434
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
237
435
 
238
436
 
239
437
  class RefreshTokenUpdater(BaseModel):
240
438
  refresh_token_name: Optional[str] = Field(
241
- 'refresh_token',
242
- description='The name of the property which contains the updated refresh token in the response from the token refresh endpoint.',
243
- examples=['refresh_token'],
244
- title='Refresh Token Property Name',
439
+ "refresh_token",
440
+ description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.",
441
+ examples=["refresh_token"],
442
+ title="Refresh Token Property Name",
245
443
  )
246
444
  access_token_config_path: Optional[List[str]] = Field(
247
- ['credentials', 'access_token'],
248
- description='Config path to the access token. Make sure the field actually exists in the config.',
249
- examples=[['credentials', 'access_token'], ['access_token']],
250
- title='Config Path To Access Token',
445
+ ["credentials", "access_token"],
446
+ description="Config path to the access token. Make sure the field actually exists in the config.",
447
+ examples=[["credentials", "access_token"], ["access_token"]],
448
+ title="Config Path To Access Token",
251
449
  )
252
450
  refresh_token_config_path: Optional[List[str]] = Field(
253
- ['credentials', 'refresh_token'],
254
- description='Config path to the access token. Make sure the field actually exists in the config.',
255
- examples=[['credentials', 'refresh_token'], ['refresh_token']],
256
- title='Config Path To Refresh Token',
451
+ ["credentials", "refresh_token"],
452
+ description="Config path to the access token. Make sure the field actually exists in the config.",
453
+ examples=[["credentials", "refresh_token"], ["refresh_token"]],
454
+ title="Config Path To Refresh Token",
257
455
  )
258
456
  token_expiry_date_config_path: Optional[List[str]] = Field(
259
- ['credentials', 'token_expiry_date'],
260
- description='Config path to the expiry date. Make sure actually exists in the config.',
261
- examples=[['credentials', 'token_expiry_date']],
262
- title='Config Path To Expiry Date',
457
+ ["credentials", "token_expiry_date"],
458
+ description="Config path to the expiry date. Make sure actually exists in the config.",
459
+ examples=[["credentials", "token_expiry_date"]],
460
+ title="Config Path To Expiry Date",
263
461
  )
264
462
  refresh_token_error_status_codes: Optional[List[int]] = Field(
265
463
  [],
266
- description='Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error',
464
+ description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error",
267
465
  examples=[[400, 500]],
268
- title='Refresh Token Error Status Codes',
466
+ title="Refresh Token Error Status Codes",
269
467
  )
270
468
  refresh_token_error_key: Optional[str] = Field(
271
- '',
272
- description='Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).',
273
- examples=['error'],
274
- title='Refresh Token Error Key',
469
+ "",
470
+ description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).",
471
+ examples=["error"],
472
+ title="Refresh Token Error Key",
275
473
  )
276
474
  refresh_token_error_values: Optional[List[str]] = Field(
277
475
  [],
278
476
  description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error',
279
- examples=[['invalid_grant', 'invalid_permissions']],
280
- title='Refresh Token Error Values',
477
+ examples=[["invalid_grant", "invalid_permissions"]],
478
+ title="Refresh Token Error Values",
281
479
  )
282
480
 
283
481
 
284
482
  class OAuthAuthenticator(BaseModel):
285
- type: Literal['OAuthAuthenticator']
483
+ type: Literal["OAuthAuthenticator"]
286
484
  client_id: str = Field(
287
485
  ...,
288
- description='The OAuth client ID. Fill it in the user inputs.',
486
+ description="The OAuth client ID. Fill it in the user inputs.",
289
487
  examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"],
290
- title='Client ID',
488
+ title="Client ID",
291
489
  )
292
490
  client_secret: str = Field(
293
491
  ...,
294
- description='The OAuth client secret. Fill it in the user inputs.',
492
+ description="The OAuth client secret. Fill it in the user inputs.",
295
493
  examples=[
296
494
  "{{ config['client_secret }}",
297
495
  "{{ config['credentials']['client_secret }}",
298
496
  ],
299
- title='Client Secret',
497
+ title="Client Secret",
300
498
  )
301
499
  refresh_token: Optional[str] = Field(
302
500
  None,
303
- description='Credential artifact used to get a new access token.',
501
+ description="Credential artifact used to get a new access token.",
304
502
  examples=[
305
503
  "{{ config['refresh_token'] }}",
306
504
  "{{ config['credentials]['refresh_token'] }}",
307
505
  ],
308
- title='Refresh Token',
506
+ title="Refresh Token",
309
507
  )
310
- token_refresh_endpoint: str = Field(
311
- ...,
312
- description='The full URL to call to obtain a new access token.',
313
- examples=['https://connect.squareup.com/oauth2/token'],
314
- title='Token Refresh Endpoint',
508
+ token_refresh_endpoint: Optional[str] = Field(
509
+ None,
510
+ description="The full URL to call to obtain a new access token.",
511
+ examples=["https://connect.squareup.com/oauth2/token"],
512
+ title="Token Refresh Endpoint",
315
513
  )
316
514
  access_token_name: Optional[str] = Field(
317
- 'access_token',
318
- description='The name of the property which contains the access token in the response from the token refresh endpoint.',
319
- examples=['access_token'],
320
- title='Access Token Property Name',
515
+ "access_token",
516
+ description="The name of the property which contains the access token in the response from the token refresh endpoint.",
517
+ examples=["access_token"],
518
+ title="Access Token Property Name",
519
+ )
520
+ access_token_value: Optional[str] = Field(
521
+ None,
522
+ description="The value of the access_token to bypass the token refreshing using `refresh_token`.",
523
+ examples=["secret_access_token_value"],
524
+ title="Access Token Value",
321
525
  )
322
526
  expires_in_name: Optional[str] = Field(
323
- 'expires_in',
324
- description='The name of the property which contains the expiry date in the response from the token refresh endpoint.',
325
- examples=['expires_in'],
326
- title='Token Expiry Property Name',
527
+ "expires_in",
528
+ description="The name of the property which contains the expiry date in the response from the token refresh endpoint.",
529
+ examples=["expires_in"],
530
+ title="Token Expiry Property Name",
327
531
  )
328
532
  grant_type: Optional[str] = Field(
329
- 'refresh_token',
330
- description='Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.',
331
- examples=['refresh_token', 'client_credentials'],
332
- title='Grant Type',
533
+ "refresh_token",
534
+ description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
535
+ examples=["refresh_token", "client_credentials"],
536
+ title="Grant Type",
333
537
  )
334
538
  refresh_request_body: Optional[Dict[str, Any]] = Field(
335
539
  None,
336
- description='Body of the request sent to get a new access token.',
540
+ description="Body of the request sent to get a new access token.",
337
541
  examples=[
338
542
  {
339
- 'applicationId': "{{ config['application_id'] }}",
340
- 'applicationSecret': "{{ config['application_secret'] }}",
341
- 'token': "{{ config['token'] }}",
543
+ "applicationId": "{{ config['application_id'] }}",
544
+ "applicationSecret": "{{ config['application_secret'] }}",
545
+ "token": "{{ config['token'] }}",
342
546
  }
343
547
  ],
344
- title='Refresh Request Body',
548
+ title="Refresh Request Body",
345
549
  )
346
550
  scopes: Optional[List[str]] = Field(
347
551
  None,
348
- description='List of scopes that should be granted to the access token.',
349
- examples=[
350
- ['crm.list.read', 'crm.objects.contacts.read', 'crm.schema.contacts.read']
351
- ],
352
- title='Scopes',
552
+ description="List of scopes that should be granted to the access token.",
553
+ examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]],
554
+ title="Scopes",
353
555
  )
354
556
  token_expiry_date: Optional[str] = Field(
355
557
  None,
356
- description='The access token expiry date.',
357
- examples=['2023-04-06T07:12:10.421833+00:00', 1680842386],
358
- title='Token Expiry Date',
558
+ description="The access token expiry date.",
559
+ examples=["2023-04-06T07:12:10.421833+00:00", 1680842386],
560
+ title="Token Expiry Date",
359
561
  )
360
562
  token_expiry_date_format: Optional[str] = Field(
361
563
  None,
362
- description='The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.',
363
- examples=['%Y-%m-%d %H:%M:%S.%f+00:00'],
364
- title='Token Expiry Date Format',
564
+ description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.",
565
+ examples=["%Y-%m-%d %H:%M:%S.%f+00:00"],
566
+ title="Token Expiry Date Format",
365
567
  )
366
568
  refresh_token_updater: Optional[RefreshTokenUpdater] = Field(
367
569
  None,
368
- description='When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.',
369
- title='Token Updater',
570
+ description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.",
571
+ title="Token Updater",
572
+ )
573
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
574
+
575
+
576
+ class DpathExtractor(BaseModel):
577
+ type: Literal["DpathExtractor"]
578
+ field_path: List[str] = Field(
579
+ ...,
580
+ description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
581
+ examples=[
582
+ ["data"],
583
+ ["data", "records"],
584
+ ["data", "{{ parameters.name }}"],
585
+ ["data", "*", "record"],
586
+ ],
587
+ title="Field Path",
370
588
  )
371
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
589
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
590
+
591
+
592
+ class ResponseToFileExtractor(BaseModel):
593
+ type: Literal["ResponseToFileExtractor"]
594
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
372
595
 
373
596
 
374
597
  class ExponentialBackoffStrategy(BaseModel):
375
- type: Literal['ExponentialBackoffStrategy']
598
+ type: Literal["ExponentialBackoffStrategy"]
376
599
  factor: Optional[Union[float, str]] = Field(
377
600
  5,
378
- description='Multiplicative constant applied on each retry.',
379
- examples=[5, 5.5, '10'],
380
- title='Factor',
601
+ description="Multiplicative constant applied on each retry.",
602
+ examples=[5, 5.5, "10"],
603
+ title="Factor",
381
604
  )
382
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
605
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
383
606
 
384
607
 
385
608
  class SessionTokenRequestBearerAuthenticator(BaseModel):
386
- type: Literal['Bearer']
609
+ type: Literal["Bearer"]
387
610
 
388
611
 
389
612
  class HttpMethod(Enum):
390
- GET = 'GET'
391
- POST = 'POST'
613
+ GET = "GET"
614
+ POST = "POST"
392
615
 
393
616
 
394
617
  class Action(Enum):
395
- SUCCESS = 'SUCCESS'
396
- FAIL = 'FAIL'
397
- RETRY = 'RETRY'
398
- IGNORE = 'IGNORE'
618
+ SUCCESS = "SUCCESS"
619
+ FAIL = "FAIL"
620
+ RETRY = "RETRY"
621
+ IGNORE = "IGNORE"
622
+ RATE_LIMITED = "RATE_LIMITED"
623
+
624
+
625
+ class FailureType(Enum):
626
+ system_error = "system_error"
627
+ config_error = "config_error"
628
+ transient_error = "transient_error"
399
629
 
400
630
 
401
631
  class HttpResponseFilter(BaseModel):
402
- type: Literal['HttpResponseFilter']
403
- action: Action = Field(
404
- ...,
405
- description='Action to execute if a response matches the filter.',
406
- examples=['SUCCESS', 'FAIL', 'RETRY', 'IGNORE'],
407
- title='Action',
632
+ type: Literal["HttpResponseFilter"]
633
+ action: Optional[Action] = Field(
634
+ None,
635
+ description="Action to execute if a response matches the filter.",
636
+ examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"],
637
+ title="Action",
638
+ )
639
+ failure_type: Optional[FailureType] = Field(
640
+ None,
641
+ description="Failure type of traced exception if a response matches the filter.",
642
+ examples=["system_error", "config_error", "transient_error"],
643
+ title="Failure Type",
408
644
  )
409
645
  error_message: Optional[str] = Field(
410
646
  None,
411
- description='Error Message to display if the response matches the filter.',
412
- title='Error Message',
647
+ description="Error Message to display if the response matches the filter.",
648
+ title="Error Message",
413
649
  )
414
650
  error_message_contains: Optional[str] = Field(
415
651
  None,
416
- description='Match the response if its error message contains the substring.',
417
- example=['This API operation is not enabled for this site'],
418
- title='Error Message Substring',
652
+ description="Match the response if its error message contains the substring.",
653
+ example=["This API operation is not enabled for this site"],
654
+ title="Error Message Substring",
419
655
  )
420
656
  http_codes: Optional[List[int]] = Field(
421
657
  None,
422
- description='Match the response if its HTTP code is included in this list.',
658
+ description="Match the response if its HTTP code is included in this list.",
423
659
  examples=[[420, 429], [500]],
424
- title='HTTP Codes',
660
+ title="HTTP Codes",
661
+ unique_items=True,
425
662
  )
426
663
  predicate: Optional[str] = Field(
427
664
  None,
428
- description='Match the response if the predicate evaluates to true.',
665
+ description="Match the response if the predicate evaluates to true.",
429
666
  examples=[
430
667
  "{{ 'Too much requests' in response }}",
431
668
  "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}",
432
669
  ],
433
- title='Predicate',
670
+ title="Predicate",
434
671
  )
435
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
672
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
673
+
674
+
675
+ class TypesMap(BaseModel):
676
+ target_type: Union[str, List[str]]
677
+ current_type: Union[str, List[str]]
678
+
679
+
680
+ class SchemaTypeIdentifier(BaseModel):
681
+ type: Optional[Literal["SchemaTypeIdentifier"]] = None
682
+ schema_pointer: Optional[List[str]] = Field(
683
+ [],
684
+ description="List of nested fields defining the schema field path to extract. Defaults to [].",
685
+ title="Schema Path",
686
+ )
687
+ key_pointer: List[str] = Field(
688
+ ...,
689
+ description="List of potentially nested fields describing the full path of the field key to extract.",
690
+ title="Key Path",
691
+ )
692
+ type_pointer: Optional[List[str]] = Field(
693
+ None,
694
+ description="List of potentially nested fields describing the full path of the field type to extract.",
695
+ title="Type Path",
696
+ )
697
+ types_mapping: Optional[List[TypesMap]] = None
698
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
436
699
 
437
700
 
438
701
  class InlineSchemaLoader(BaseModel):
439
- type: Literal['InlineSchemaLoader']
702
+ type: Literal["InlineSchemaLoader"]
440
703
  schema_: Optional[Dict[str, Any]] = Field(
441
704
  None,
442
- alias='schema',
705
+ alias="schema",
443
706
  description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.',
444
- title='Schema',
707
+ title="Schema",
445
708
  )
446
709
 
447
710
 
448
711
  class JsonFileSchemaLoader(BaseModel):
449
- type: Literal['JsonFileSchemaLoader']
712
+ type: Literal["JsonFileSchemaLoader"]
450
713
  file_path: Optional[str] = Field(
451
714
  None,
452
715
  description="Path to the JSON file defining the schema. The path is relative to the connector module's root.",
453
- example=['./schemas/users.json'],
454
- title='File Path',
716
+ example=["./schemas/users.json"],
717
+ title="File Path",
455
718
  )
456
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
719
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
457
720
 
458
721
 
459
722
  class JsonDecoder(BaseModel):
460
- type: Literal['JsonDecoder']
723
+ type: Literal["JsonDecoder"]
724
+
725
+
726
+ class JsonlDecoder(BaseModel):
727
+ type: Literal["JsonlDecoder"]
728
+
729
+
730
+ class KeysToLower(BaseModel):
731
+ type: Literal["KeysToLower"]
732
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
733
+
734
+
735
+ class KeysToSnakeCase(BaseModel):
736
+ type: Literal["KeysToSnakeCase"]
737
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
738
+
739
+
740
+ class KeysReplace(BaseModel):
741
+ type: Literal["KeysReplace"]
742
+ old: str = Field(
743
+ ...,
744
+ description="Old value to replace.",
745
+ examples=[" ", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
746
+ title="Old value",
747
+ )
748
+ new: str = Field(
749
+ ...,
750
+ description="New value to set.",
751
+ examples=["_", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
752
+ title="New value",
753
+ )
754
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
755
+
756
+
757
+ class FlattenFields(BaseModel):
758
+ type: Literal["FlattenFields"]
759
+ flatten_lists: Optional[bool] = Field(
760
+ True,
761
+ description="Whether to flatten lists or leave it as is. Default is True.",
762
+ title="Flatten Lists",
763
+ )
764
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
765
+
766
+
767
+ class IterableDecoder(BaseModel):
768
+ type: Literal["IterableDecoder"]
769
+
770
+
771
+ class XmlDecoder(BaseModel):
772
+ type: Literal["XmlDecoder"]
773
+
774
+
775
+ class CustomDecoder(BaseModel):
776
+ class Config:
777
+ extra = Extra.allow
778
+
779
+ type: Literal["CustomDecoder"]
780
+ class_name: str = Field(
781
+ ...,
782
+ description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.",
783
+ examples=["source_amazon_ads.components.GzipJsonlDecoder"],
784
+ title="Class Name",
785
+ )
786
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
787
+
788
+
789
+ class GzipJsonDecoder(BaseModel):
790
+ class Config:
791
+ extra = Extra.allow
792
+
793
+ type: Literal["GzipJsonDecoder"]
794
+ encoding: Optional[str] = "utf-8"
795
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
461
796
 
462
797
 
463
798
  class MinMaxDatetime(BaseModel):
464
- type: Literal['MinMaxDatetime']
799
+ type: Literal["MinMaxDatetime"]
465
800
  datetime: str = Field(
466
801
  ...,
467
- description='Datetime value.',
468
- examples=['2021-01-01', '2021-01-01T00:00:00Z', "{{ config['start_time'] }}"],
469
- title='Datetime',
802
+ description="Datetime value.",
803
+ examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"],
804
+ title="Datetime",
470
805
  )
471
806
  datetime_format: Optional[str] = Field(
472
- '',
473
- description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
474
- examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s'],
475
- title='Datetime Format',
807
+ "",
808
+ description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
809
+ examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
810
+ title="Datetime Format",
476
811
  )
477
812
  max_datetime: Optional[str] = Field(
478
813
  None,
479
- description='Ceiling applied on the datetime value. Must be formatted with the datetime_format field.',
480
- examples=['2021-01-01T00:00:00Z', '2021-01-01'],
481
- title='Max Datetime',
814
+ description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.",
815
+ examples=["2021-01-01T00:00:00Z", "2021-01-01"],
816
+ title="Max Datetime",
482
817
  )
483
818
  min_datetime: Optional[str] = Field(
484
819
  None,
485
- description='Floor applied on the datetime value. Must be formatted with the datetime_format field.',
486
- examples=['2010-01-01T00:00:00Z', '2010-01-01'],
487
- title='Min Datetime',
820
+ description="Floor applied on the datetime value. Must be formatted with the datetime_format field.",
821
+ examples=["2010-01-01T00:00:00Z", "2010-01-01"],
822
+ title="Min Datetime",
488
823
  )
489
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
824
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
490
825
 
491
826
 
492
827
  class NoAuth(BaseModel):
493
- type: Literal['NoAuth']
494
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
828
+ type: Literal["NoAuth"]
829
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
495
830
 
496
831
 
497
832
  class NoPagination(BaseModel):
498
- type: Literal['NoPagination']
833
+ type: Literal["NoPagination"]
834
+
835
+
836
+ class State(BaseModel):
837
+ class Config:
838
+ extra = Extra.allow
839
+
840
+ min: int
841
+ max: int
842
+
843
+
844
+ class OauthConnectorInputSpecification(BaseModel):
845
+ class Config:
846
+ extra = Extra.allow
847
+
848
+ consent_url: str = Field(
849
+ ...,
850
+ description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
851
+ examples=[
852
+ "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
853
+ "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}",
854
+ ],
855
+ title="Consent URL",
856
+ )
857
+ scope: Optional[str] = Field(
858
+ None,
859
+ description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
860
+ examples=["user:read user:read_orders workspaces:read"],
861
+ title="Scopes",
862
+ )
863
+ access_token_url: str = Field(
864
+ ...,
865
+ description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
866
+ examples=[
867
+ "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
868
+ ],
869
+ title="Access Token URL",
870
+ )
871
+ access_token_headers: Optional[Dict[str, Any]] = Field(
872
+ None,
873
+ description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
874
+ examples=[{"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"}],
875
+ title="Access Token Headers",
876
+ )
877
+ access_token_params: Optional[Dict[str, Any]] = Field(
878
+ None,
879
+ description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
880
+ examples=[
881
+ {
882
+ "{auth_code_key}": "{{auth_code_key}}",
883
+ "{client_id_key}": "{{client_id_key}}",
884
+ "{client_secret_key}": "{{client_secret_key}}",
885
+ }
886
+ ],
887
+ title="Access Token Query Params (Json Encoded)",
888
+ )
889
+ extract_output: List[str] = Field(
890
+ ...,
891
+ description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.",
892
+ examples=[["access_token", "refresh_token", "other_field"]],
893
+ title="Extract Output",
894
+ )
895
+ state: Optional[State] = Field(
896
+ None,
897
+ description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.",
898
+ examples=[{"min": 7, "max": 128}],
899
+ title="Configurable State Query Param",
900
+ )
901
+ client_id_key: Optional[str] = Field(
902
+ None,
903
+ description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
904
+ examples=["my_custom_client_id_key_name"],
905
+ title="Client ID Key Override",
906
+ )
907
+ client_secret_key: Optional[str] = Field(
908
+ None,
909
+ description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
910
+ examples=["my_custom_client_secret_key_name"],
911
+ title="Client Secret Key Override",
912
+ )
913
+ scope_key: Optional[str] = Field(
914
+ None,
915
+ description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
916
+ examples=["my_custom_scope_key_key_name"],
917
+ title="Scopes Key Override",
918
+ )
919
+ state_key: Optional[str] = Field(
920
+ None,
921
+ description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.",
922
+ examples=["my_custom_state_key_key_name"],
923
+ title="State Key Override",
924
+ )
925
+ auth_code_key: Optional[str] = Field(
926
+ None,
927
+ description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.",
928
+ examples=["my_custom_auth_code_key_name"],
929
+ title="Auth Code Key Override",
930
+ )
931
+ redirect_uri_key: Optional[str] = Field(
932
+ None,
933
+ description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
934
+ examples=["my_custom_redirect_uri_key_name"],
935
+ title="Redirect URI Key Override",
936
+ )
499
937
 
500
938
 
501
939
  class OAuthConfigSpecification(BaseModel):
502
940
  class Config:
503
941
  extra = Extra.allow
504
942
 
505
- oauth_user_input_from_connector_config_specification: Optional[
506
- Dict[str, Any]
507
- ] = Field(
943
+ oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field(
508
944
  None,
509
945
  description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }",
510
946
  examples=[
511
- {'app_id': {'type': 'string', 'path_in_connector_config': ['app_id']}},
947
+ {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}},
512
948
  {
513
- 'app_id': {
514
- 'type': 'string',
515
- 'path_in_connector_config': ['info', 'app_id'],
949
+ "app_id": {
950
+ "type": "string",
951
+ "path_in_connector_config": ["info", "app_id"],
516
952
  }
517
953
  },
518
954
  ],
519
- title='OAuth user input',
955
+ title="OAuth user input",
956
+ )
957
+ oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
958
+ None,
959
+ description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{auth_code_key}": "{{auth_code_key}}",\n "{client_id_key}": "{{client_id_key}}",\n "{client_secret_key}": "{{client_secret_key}}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }',
960
+ title="DeclarativeOAuth Connector Specification",
520
961
  )
521
962
  complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
522
963
  None,
523
964
  description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
524
965
  examples=[
525
966
  {
526
- 'refresh_token': {
527
- 'type': 'string,',
528
- 'path_in_connector_config': ['credentials', 'refresh_token'],
967
+ "refresh_token": {
968
+ "type": "string,",
969
+ "path_in_connector_config": ["credentials", "refresh_token"],
529
970
  }
530
971
  }
531
972
  ],
532
- title='OAuth output specification',
973
+ title="OAuth output specification",
533
974
  )
534
975
  complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field(
535
976
  None,
536
- description='OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }',
537
- examples=[
538
- {'client_id': {'type': 'string'}, 'client_secret': {'type': 'string'}}
539
- ],
540
- title='OAuth input specification',
977
+ description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }",
978
+ examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}],
979
+ title="OAuth input specification",
541
980
  )
542
981
  complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field(
543
982
  None,
544
983
  description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }",
545
984
  examples=[
546
985
  {
547
- 'client_id': {
548
- 'type': 'string,',
549
- 'path_in_connector_config': ['credentials', 'client_id'],
986
+ "client_id": {
987
+ "type": "string,",
988
+ "path_in_connector_config": ["credentials", "client_id"],
550
989
  },
551
- 'client_secret': {
552
- 'type': 'string,',
553
- 'path_in_connector_config': ['credentials', 'client_secret'],
990
+ "client_secret": {
991
+ "type": "string,",
992
+ "path_in_connector_config": ["credentials", "client_secret"],
554
993
  },
555
994
  }
556
995
  ],
557
- title='OAuth server output specification',
996
+ title="OAuth server output specification",
558
997
  )
559
998
 
560
999
 
561
1000
  class OffsetIncrement(BaseModel):
562
- type: Literal['OffsetIncrement']
1001
+ type: Literal["OffsetIncrement"]
563
1002
  page_size: Optional[Union[int, str]] = Field(
564
1003
  None,
565
- description='The number of records to include in each pages.',
1004
+ description="The number of records to include in each pages.",
566
1005
  examples=[100, "{{ config['page_size'] }}"],
567
- title='Limit',
1006
+ title="Limit",
568
1007
  )
569
1008
  inject_on_first_request: Optional[bool] = Field(
570
1009
  False,
571
- description='Using the `offset` with value `0` during the first request',
572
- title='Inject Offset',
1010
+ description="Using the `offset` with value `0` during the first request",
1011
+ title="Inject Offset",
573
1012
  )
574
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1013
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
575
1014
 
576
1015
 
577
1016
  class PageIncrement(BaseModel):
578
- type: Literal['PageIncrement']
1017
+ type: Literal["PageIncrement"]
579
1018
  page_size: Optional[Union[int, str]] = Field(
580
1019
  None,
581
- description='The number of records to include in each pages.',
582
- examples=[100, '100', "{{ config['page_size'] }}"],
583
- title='Page Size',
1020
+ description="The number of records to include in each pages.",
1021
+ examples=[100, "100", "{{ config['page_size'] }}"],
1022
+ title="Page Size",
584
1023
  )
585
1024
  start_from_page: Optional[int] = Field(
586
1025
  0,
587
- description='Index of the first page to request.',
1026
+ description="Index of the first page to request.",
588
1027
  examples=[0, 1],
589
- title='Start From Page',
1028
+ title="Start From Page",
590
1029
  )
591
1030
  inject_on_first_request: Optional[bool] = Field(
592
1031
  False,
593
- description='Using the `page number` with value defined by `start_from_page` during the first request',
594
- title='Inject Page Number',
1032
+ description="Using the `page number` with value defined by `start_from_page` during the first request",
1033
+ title="Inject Page Number",
595
1034
  )
596
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1035
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
597
1036
 
598
1037
 
599
1038
  class PrimaryKey(BaseModel):
600
1039
  __root__: Union[str, List[str], List[List[str]]] = Field(
601
1040
  ...,
602
- description='The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.',
603
- examples=['id', ['code', 'type']],
604
- title='Primary Key',
1041
+ description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.",
1042
+ examples=["id", ["code", "type"]],
1043
+ title="Primary Key",
605
1044
  )
606
1045
 
607
1046
 
608
1047
  class RecordFilter(BaseModel):
609
- type: Literal['RecordFilter']
1048
+ type: Literal["RecordFilter"]
610
1049
  condition: Optional[str] = Field(
611
- '',
612
- description='The predicate to filter a record. Records will be removed if evaluated to False.',
1050
+ "",
1051
+ description="The predicate to filter a record. Records will be removed if evaluated to False.",
613
1052
  examples=[
614
1053
  "{{ record['created_at'] >= stream_interval['start_time'] }}",
615
1054
  "{{ record.status in ['active', 'expired'] }}",
616
1055
  ],
617
1056
  )
618
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1057
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
619
1058
 
620
1059
 
621
1060
  class SchemaNormalization(Enum):
622
- None_ = 'None'
623
- Default = 'Default'
1061
+ None_ = "None"
1062
+ Default = "Default"
624
1063
 
625
1064
 
626
1065
  class RemoveFields(BaseModel):
627
- type: Literal['RemoveFields']
1066
+ type: Literal["RemoveFields"]
628
1067
  condition: Optional[str] = Field(
629
- '',
630
- description='The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,',
1068
+ "",
1069
+ description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,",
631
1070
  examples=[
632
1071
  "{{ property|string == '' }}",
633
- '{{ property is integer }}',
634
- '{{ property|length > 5 }}',
1072
+ "{{ property is integer }}",
1073
+ "{{ property|length > 5 }}",
635
1074
  "{{ property == 'some_string_to_match' }}",
636
1075
  ],
637
1076
  )
638
1077
  field_pointers: List[List[str]] = Field(
639
1078
  ...,
640
- description='Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.',
641
- examples=[['tags'], [['content', 'html'], ['content', 'plain_text']]],
642
- title='Field Paths',
1079
+ description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.",
1080
+ examples=[["tags"], [["content", "html"], ["content", "plain_text"]]],
1081
+ title="Field Paths",
643
1082
  )
644
1083
 
645
1084
 
646
1085
  class RequestPath(BaseModel):
647
- type: Literal['RequestPath']
1086
+ type: Literal["RequestPath"]
648
1087
 
649
1088
 
650
1089
  class InjectInto(Enum):
651
- request_parameter = 'request_parameter'
652
- header = 'header'
653
- body_data = 'body_data'
654
- body_json = 'body_json'
1090
+ request_parameter = "request_parameter"
1091
+ header = "header"
1092
+ body_data = "body_data"
1093
+ body_json = "body_json"
655
1094
 
656
1095
 
657
1096
  class RequestOption(BaseModel):
658
- type: Literal['RequestOption']
1097
+ type: Literal["RequestOption"]
659
1098
  field_name: str = Field(
660
1099
  ...,
661
- description='Configures which key should be used in the location that the descriptor is being injected into',
662
- examples=['segment_id'],
663
- title='Request Option',
1100
+ description="Configures which key should be used in the location that the descriptor is being injected into",
1101
+ examples=["segment_id"],
1102
+ title="Request Option",
664
1103
  )
665
1104
  inject_into: InjectInto = Field(
666
1105
  ...,
667
- description='Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.',
668
- examples=['request_parameter', 'header', 'body_data', 'body_json'],
669
- title='Inject Into',
1106
+ description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.",
1107
+ examples=["request_parameter", "header", "body_data", "body_json"],
1108
+ title="Inject Into",
670
1109
  )
671
1110
 
672
1111
 
@@ -678,106 +1117,184 @@ class Schemas(BaseModel):
678
1117
 
679
1118
 
680
1119
  class LegacySessionTokenAuthenticator(BaseModel):
681
- type: Literal['LegacySessionTokenAuthenticator']
1120
+ type: Literal["LegacySessionTokenAuthenticator"]
682
1121
  header: str = Field(
683
1122
  ...,
684
- description='The name of the session token header that will be injected in the request',
685
- examples=['X-Session'],
686
- title='Session Request Header',
1123
+ description="The name of the session token header that will be injected in the request",
1124
+ examples=["X-Session"],
1125
+ title="Session Request Header",
687
1126
  )
688
1127
  login_url: str = Field(
689
1128
  ...,
690
- description='Path of the login URL (do not include the base URL)',
691
- examples=['session'],
692
- title='Login Path',
1129
+ description="Path of the login URL (do not include the base URL)",
1130
+ examples=["session"],
1131
+ title="Login Path",
693
1132
  )
694
1133
  session_token: Optional[str] = Field(
695
1134
  None,
696
- description='Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair',
1135
+ description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair",
697
1136
  example=["{{ config['session_token'] }}"],
698
- title='Session Token',
1137
+ title="Session Token",
699
1138
  )
700
1139
  session_token_response_key: str = Field(
701
1140
  ...,
702
- description='Name of the key of the session token to be extracted from the response',
703
- examples=['id'],
704
- title='Response Token Response Key',
1141
+ description="Name of the key of the session token to be extracted from the response",
1142
+ examples=["id"],
1143
+ title="Response Token Response Key",
705
1144
  )
706
1145
  username: Optional[str] = Field(
707
1146
  None,
708
- description='Username used to authenticate and obtain a session token',
1147
+ description="Username used to authenticate and obtain a session token",
709
1148
  examples=[" {{ config['username'] }}"],
710
- title='Username',
1149
+ title="Username",
711
1150
  )
712
1151
  password: Optional[str] = Field(
713
- '',
714
- description='Password used to authenticate and obtain a session token',
715
- examples=["{{ config['password'] }}", ''],
716
- title='Password',
1152
+ "",
1153
+ description="Password used to authenticate and obtain a session token",
1154
+ examples=["{{ config['password'] }}", ""],
1155
+ title="Password",
717
1156
  )
718
1157
  validate_session_url: str = Field(
719
1158
  ...,
720
- description='Path of the URL to use to validate that the session token is valid (do not include the base URL)',
721
- examples=['user/current'],
722
- title='Validate Session Path',
1159
+ description="Path of the URL to use to validate that the session token is valid (do not include the base URL)",
1160
+ examples=["user/current"],
1161
+ title="Validate Session Path",
723
1162
  )
724
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1163
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1164
+
1165
+
1166
+ class JsonLineParser(BaseModel):
1167
+ type: Literal["JsonLineParser"]
1168
+ encoding: Optional[str] = "utf-8"
1169
+
1170
+
1171
+ class CsvParser(BaseModel):
1172
+ type: Literal["CsvParser"]
1173
+ encoding: Optional[str] = "utf-8"
1174
+ delimiter: Optional[str] = ","
1175
+
1176
+
1177
+ class AsyncJobStatusMap(BaseModel):
1178
+ type: Optional[Literal["AsyncJobStatusMap"]] = None
1179
+ running: List[str]
1180
+ completed: List[str]
1181
+ failed: List[str]
1182
+ timeout: List[str]
725
1183
 
726
1184
 
727
1185
  class ValueType(Enum):
728
- string = 'string'
729
- number = 'number'
730
- integer = 'integer'
731
- boolean = 'boolean'
1186
+ string = "string"
1187
+ number = "number"
1188
+ integer = "integer"
1189
+ boolean = "boolean"
732
1190
 
733
1191
 
734
1192
  class WaitTimeFromHeader(BaseModel):
735
- type: Literal['WaitTimeFromHeader']
1193
+ type: Literal["WaitTimeFromHeader"]
736
1194
  header: str = Field(
737
1195
  ...,
738
- description='The name of the response header defining how long to wait before retrying.',
739
- examples=['Retry-After'],
740
- title='Response Header Name',
1196
+ description="The name of the response header defining how long to wait before retrying.",
1197
+ examples=["Retry-After"],
1198
+ title="Response Header Name",
741
1199
  )
742
1200
  regex: Optional[str] = Field(
743
1201
  None,
744
- description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.',
745
- examples=['([-+]?\\d+)'],
746
- title='Extraction Regex',
1202
+ description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
1203
+ examples=["([-+]?\\d+)"],
1204
+ title="Extraction Regex",
747
1205
  )
748
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1206
+ max_waiting_time_in_seconds: Optional[float] = Field(
1207
+ None,
1208
+ description="Given the value extracted from the header is greater than this value, stop the stream.",
1209
+ examples=[3600],
1210
+ title="Max Waiting Time in Seconds",
1211
+ )
1212
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
749
1213
 
750
1214
 
751
1215
  class WaitUntilTimeFromHeader(BaseModel):
752
- type: Literal['WaitUntilTimeFromHeader']
1216
+ type: Literal["WaitUntilTimeFromHeader"]
753
1217
  header: str = Field(
754
1218
  ...,
755
- description='The name of the response header defining how long to wait before retrying.',
756
- examples=['wait_time'],
757
- title='Response Header',
1219
+ description="The name of the response header defining how long to wait before retrying.",
1220
+ examples=["wait_time"],
1221
+ title="Response Header",
758
1222
  )
759
1223
  min_wait: Optional[Union[float, str]] = Field(
760
1224
  None,
761
- description='Minimum time to wait before retrying.',
762
- examples=[10, '60'],
763
- title='Minimum Wait Time',
1225
+ description="Minimum time to wait before retrying.",
1226
+ examples=[10, "60"],
1227
+ title="Minimum Wait Time",
764
1228
  )
765
1229
  regex: Optional[str] = Field(
766
1230
  None,
767
- description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.',
768
- examples=['([-+]?\\d+)'],
769
- title='Extraction Regex',
1231
+ description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
1232
+ examples=["([-+]?\\d+)"],
1233
+ title="Extraction Regex",
770
1234
  )
771
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1235
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1236
+
1237
+
1238
+ class ComponentMappingDefinition(BaseModel):
1239
+ type: Literal["ComponentMappingDefinition"]
1240
+ field_path: List[str] = Field(
1241
+ ...,
1242
+ description="A list of potentially nested fields indicating the full path where value will be added or updated.",
1243
+ examples=[
1244
+ ["data"],
1245
+ ["data", "records"],
1246
+ ["data", 1, "name"],
1247
+ ["data", "{{ components_values.name }}"],
1248
+ ["data", "*", "record"],
1249
+ ["*", "**", "name"],
1250
+ ],
1251
+ title="Field Path",
1252
+ )
1253
+ value: str = Field(
1254
+ ...,
1255
+ description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.",
1256
+ examples=[
1257
+ "{{ components_values['updates'] }}",
1258
+ "{{ components_values['MetaData']['LastUpdatedTime'] }}",
1259
+ "{{ config['segment_id'] }}",
1260
+ "{{ stream_slice['parent_id'] }}",
1261
+ "{{ stream_slice['extra_fields']['name'] }}",
1262
+ ],
1263
+ title="Value",
1264
+ )
1265
+ value_type: Optional[ValueType] = Field(
1266
+ None,
1267
+ description="The expected data type of the value. If omitted, the type will be inferred from the value provided.",
1268
+ title="Value Type",
1269
+ )
1270
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1271
+
1272
+
1273
+ class StreamConfig(BaseModel):
1274
+ type: Literal["StreamConfig"]
1275
+ configs_pointer: List[str] = Field(
1276
+ ...,
1277
+ description="A list of potentially nested fields indicating the full path in source config file where streams configs located.",
1278
+ examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]],
1279
+ title="Configs Pointer",
1280
+ )
1281
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1282
+
1283
+
1284
+ class ConfigComponentsResolver(BaseModel):
1285
+ type: Literal["ConfigComponentsResolver"]
1286
+ stream_config: StreamConfig
1287
+ components_mapping: List[ComponentMappingDefinition]
1288
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
772
1289
 
773
1290
 
774
1291
  class AddedFieldDefinition(BaseModel):
775
- type: Literal['AddedFieldDefinition']
1292
+ type: Literal["AddedFieldDefinition"]
776
1293
  path: List[str] = Field(
777
1294
  ...,
778
- description='List of strings defining the path where to add the value on the record.',
779
- examples=[['segment_id'], ['metadata', 'segment_id']],
780
- title='Path',
1295
+ description="List of strings defining the path where to add the value on the record.",
1296
+ examples=[["segment_id"], ["metadata", "segment_id"]],
1297
+ title="Path",
781
1298
  )
782
1299
  value: str = Field(
783
1300
  ...,
@@ -787,187 +1304,167 @@ class AddedFieldDefinition(BaseModel):
787
1304
  "{{ record['MetaData']['LastUpdatedTime'] }}",
788
1305
  "{{ stream_partition['segment_id'] }}",
789
1306
  ],
790
- title='Value',
1307
+ title="Value",
791
1308
  )
792
1309
  value_type: Optional[ValueType] = Field(
793
1310
  None,
794
- description='Type of the value. If not specified, the type will be inferred from the value.',
795
- title='Value Type',
1311
+ description="Type of the value. If not specified, the type will be inferred from the value.",
1312
+ title="Value Type",
796
1313
  )
797
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1314
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
798
1315
 
799
1316
 
800
1317
  class AddFields(BaseModel):
801
- type: Literal['AddFields']
1318
+ type: Literal["AddFields"]
802
1319
  fields: List[AddedFieldDefinition] = Field(
803
1320
  ...,
804
- description='List of transformations (path and corresponding value) that will be added to the record.',
805
- title='Fields',
1321
+ description="List of transformations (path and corresponding value) that will be added to the record.",
1322
+ title="Fields",
806
1323
  )
807
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1324
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
808
1325
 
809
1326
 
810
1327
  class ApiKeyAuthenticator(BaseModel):
811
- type: Literal['ApiKeyAuthenticator']
1328
+ type: Literal["ApiKeyAuthenticator"]
812
1329
  api_token: Optional[str] = Field(
813
1330
  None,
814
- description='The API key to inject in the request. Fill it in the user inputs.',
1331
+ description="The API key to inject in the request. Fill it in the user inputs.",
815
1332
  examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"],
816
- title='API Key',
1333
+ title="API Key",
817
1334
  )
818
1335
  header: Optional[str] = Field(
819
1336
  None,
820
- description='The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.',
821
- examples=['Authorization', 'Api-Token', 'X-Auth-Token'],
822
- title='Header Name',
1337
+ description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.",
1338
+ examples=["Authorization", "Api-Token", "X-Auth-Token"],
1339
+ title="Header Name",
823
1340
  )
824
1341
  inject_into: Optional[RequestOption] = Field(
825
1342
  None,
826
- description='Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.',
1343
+ description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.",
827
1344
  examples=[
828
- {'inject_into': 'header', 'field_name': 'Authorization'},
829
- {'inject_into': 'request_parameter', 'field_name': 'authKey'},
1345
+ {"inject_into": "header", "field_name": "Authorization"},
1346
+ {"inject_into": "request_parameter", "field_name": "authKey"},
830
1347
  ],
831
- title='Inject API Key Into Outgoing HTTP Request',
1348
+ title="Inject API Key Into Outgoing HTTP Request",
832
1349
  )
833
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1350
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
834
1351
 
835
1352
 
836
1353
  class AuthFlow(BaseModel):
837
1354
  auth_flow_type: Optional[AuthFlowType] = Field(
838
- None, description='The type of auth to use', title='Auth flow type'
1355
+ None, description="The type of auth to use", title="Auth flow type"
839
1356
  )
840
1357
  predicate_key: Optional[List[str]] = Field(
841
1358
  None,
842
- description='JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.',
843
- examples=[['credentials', 'auth_type']],
844
- title='Predicate key',
1359
+ description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.",
1360
+ examples=[["credentials", "auth_type"]],
1361
+ title="Predicate key",
845
1362
  )
846
1363
  predicate_value: Optional[str] = Field(
847
1364
  None,
848
- description='Value of the predicate_key fields for the advanced auth to be applicable.',
849
- examples=['Oauth'],
850
- title='Predicate value',
1365
+ description="Value of the predicate_key fields for the advanced auth to be applicable.",
1366
+ examples=["Oauth"],
1367
+ title="Predicate value",
851
1368
  )
852
1369
  oauth_config_specification: Optional[OAuthConfigSpecification] = None
853
1370
 
854
1371
 
855
- class CursorPagination(BaseModel):
856
- type: Literal['CursorPagination']
857
- cursor_value: str = Field(
858
- ...,
859
- description='Value of the cursor defining the next page to fetch.',
860
- examples=[
861
- '{{ headers.link.next.cursor }}',
862
- "{{ last_records[-1]['key'] }}",
863
- "{{ response['nextPage'] }}",
864
- ],
865
- title='Cursor Value',
866
- )
867
- page_size: Optional[int] = Field(
868
- None,
869
- description='The number of records to include in each pages.',
870
- examples=[100],
871
- title='Page Size',
872
- )
873
- stop_condition: Optional[str] = Field(
874
- None,
875
- description='Template string evaluating when to stop paginating.',
876
- examples=[
877
- '{{ response.data.has_more is false }}',
878
- "{{ 'next' not in headers['link'] }}",
879
- ],
880
- title='Stop Condition',
881
- )
882
- decoder: Optional[JsonDecoder] = Field(
883
- None,
884
- description='Component decoding the response so records can be extracted.',
885
- title='Decoder',
886
- )
887
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
888
-
889
-
890
1372
  class DatetimeBasedCursor(BaseModel):
891
- type: Literal['DatetimeBasedCursor']
1373
+ type: Literal["DatetimeBasedCursor"]
892
1374
  cursor_field: str = Field(
893
1375
  ...,
894
- description='The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.',
895
- examples=['created_at', "{{ config['record_cursor'] }}"],
896
- title='Cursor Field',
1376
+ description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.",
1377
+ examples=["created_at", "{{ config['record_cursor'] }}"],
1378
+ title="Cursor Field",
897
1379
  )
898
1380
  datetime_format: str = Field(
899
1381
  ...,
900
- description='The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
901
- examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s', '%ms'],
902
- title='Outgoing Datetime Format',
1382
+ description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
1383
+ examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"],
1384
+ title="Outgoing Datetime Format",
903
1385
  )
904
1386
  start_datetime: Union[str, MinMaxDatetime] = Field(
905
1387
  ...,
906
- description='The datetime that determines the earliest record that should be synced.',
907
- examples=['2020-01-1T00:00:00Z', "{{ config['start_time'] }}"],
908
- title='Start Datetime',
1388
+ description="The datetime that determines the earliest record that should be synced.",
1389
+ examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"],
1390
+ title="Start Datetime",
909
1391
  )
910
1392
  cursor_datetime_formats: Optional[List[str]] = Field(
911
1393
  None,
912
- description='The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.',
913
- title='Cursor Datetime Formats',
1394
+ description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.",
1395
+ title="Cursor Datetime Formats",
914
1396
  )
915
1397
  cursor_granularity: Optional[str] = Field(
916
1398
  None,
917
- description='Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.',
918
- examples=['PT1S'],
919
- title='Cursor Granularity',
1399
+ description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.",
1400
+ examples=["PT1S"],
1401
+ title="Cursor Granularity",
920
1402
  )
921
1403
  end_datetime: Optional[Union[str, MinMaxDatetime]] = Field(
922
1404
  None,
923
- description='The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.',
924
- examples=['2021-01-1T00:00:00Z', '{{ now_utc() }}', '{{ day_delta(-1) }}'],
925
- title='End Datetime',
1405
+ description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.",
1406
+ examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"],
1407
+ title="End Datetime",
926
1408
  )
927
1409
  end_time_option: Optional[RequestOption] = Field(
928
1410
  None,
929
- description='Optionally configures how the end datetime will be sent in requests to the source API.',
930
- title='Inject End Time Into Outgoing HTTP Request',
1411
+ description="Optionally configures how the end datetime will be sent in requests to the source API.",
1412
+ title="Inject End Time Into Outgoing HTTP Request",
931
1413
  )
932
1414
  is_data_feed: Optional[bool] = Field(
933
1415
  None,
934
- description='A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.',
935
- title='Whether the target API is formatted as a data feed',
1416
+ description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.",
1417
+ title="Whether the target API is formatted as a data feed",
1418
+ )
1419
+ is_client_side_incremental: Optional[bool] = Field(
1420
+ None,
1421
+ description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.",
1422
+ title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)",
1423
+ )
1424
+ is_compare_strictly: Optional[bool] = Field(
1425
+ False,
1426
+ description="Set to True if the target API does not accept queries where the start time equal the end time.",
1427
+ title="Whether to skip requests if the start time equals the end time",
1428
+ )
1429
+ global_substream_cursor: Optional[bool] = Field(
1430
+ False,
1431
+ description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).",
1432
+ title="Whether to store cursor as one value instead of per partition",
936
1433
  )
937
1434
  lookback_window: Optional[str] = Field(
938
1435
  None,
939
- description='Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.',
940
- examples=['P1D', "P{{ config['lookback_days'] }}D"],
941
- title='Lookback Window',
1436
+ description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.",
1437
+ examples=["P1D", "P{{ config['lookback_days'] }}D"],
1438
+ title="Lookback Window",
942
1439
  )
943
1440
  partition_field_end: Optional[str] = Field(
944
1441
  None,
945
- description='Name of the partition start time field.',
946
- examples=['ending_time'],
947
- title='Partition Field End',
1442
+ description="Name of the partition start time field.",
1443
+ examples=["ending_time"],
1444
+ title="Partition Field End",
948
1445
  )
949
1446
  partition_field_start: Optional[str] = Field(
950
1447
  None,
951
- description='Name of the partition end time field.',
952
- examples=['starting_time'],
953
- title='Partition Field Start',
1448
+ description="Name of the partition end time field.",
1449
+ examples=["starting_time"],
1450
+ title="Partition Field Start",
954
1451
  )
955
1452
  start_time_option: Optional[RequestOption] = Field(
956
1453
  None,
957
- description='Optionally configures how the start datetime will be sent in requests to the source API.',
958
- title='Inject Start Time Into Outgoing HTTP Request',
1454
+ description="Optionally configures how the start datetime will be sent in requests to the source API.",
1455
+ title="Inject Start Time Into Outgoing HTTP Request",
959
1456
  )
960
1457
  step: Optional[str] = Field(
961
1458
  None,
962
- description='The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.',
963
- examples=['P1W', "{{ config['step_increment'] }}"],
964
- title='Step',
1459
+ description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.",
1460
+ examples=["P1W", "{{ config['step_increment'] }}"],
1461
+ title="Step",
965
1462
  )
966
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1463
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
967
1464
 
968
1465
 
969
1466
  class DefaultErrorHandler(BaseModel):
970
- type: Literal['DefaultErrorHandler']
1467
+ type: Literal["DefaultErrorHandler"]
971
1468
  backoff_strategies: Optional[
972
1469
  List[
973
1470
  Union[
@@ -980,154 +1477,188 @@ class DefaultErrorHandler(BaseModel):
980
1477
  ]
981
1478
  ] = Field(
982
1479
  None,
983
- description='List of backoff strategies to use to determine how long to wait before retrying a retryable request.',
984
- title='Backoff Strategies',
1480
+ description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.",
1481
+ title="Backoff Strategies",
985
1482
  )
986
1483
  max_retries: Optional[int] = Field(
987
1484
  5,
988
- description='The maximum number of time to retry a retryable request before giving up and failing.',
1485
+ description="The maximum number of time to retry a retryable request before giving up and failing.",
989
1486
  examples=[5, 0, 10],
990
- title='Max Retry Count',
1487
+ title="Max Retry Count",
991
1488
  )
992
1489
  response_filters: Optional[List[HttpResponseFilter]] = Field(
993
1490
  None,
994
1491
  description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.",
995
- title='Response Filters',
1492
+ title="Response Filters",
996
1493
  )
997
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1494
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
998
1495
 
999
1496
 
1000
1497
  class DefaultPaginator(BaseModel):
1001
- type: Literal['DefaultPaginator']
1498
+ type: Literal["DefaultPaginator"]
1002
1499
  pagination_strategy: Union[
1003
1500
  CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement
1004
1501
  ] = Field(
1005
1502
  ...,
1006
- description='Strategy defining how records are paginated.',
1007
- title='Pagination Strategy',
1008
- )
1009
- decoder: Optional[JsonDecoder] = Field(
1010
- None,
1011
- description='Component decoding the response so records can be extracted.',
1012
- title='Decoder',
1503
+ description="Strategy defining how records are paginated.",
1504
+ title="Pagination Strategy",
1013
1505
  )
1014
1506
  page_size_option: Optional[RequestOption] = None
1015
1507
  page_token_option: Optional[Union[RequestOption, RequestPath]] = None
1016
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1017
-
1018
-
1019
- class DpathExtractor(BaseModel):
1020
- type: Literal['DpathExtractor']
1021
- field_path: List[str] = Field(
1022
- ...,
1023
- description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
1024
- examples=[
1025
- ['data'],
1026
- ['data', 'records'],
1027
- ['data', '{{ parameters.name }}'],
1028
- ['data', '*', 'record'],
1029
- ],
1030
- title='Field Path',
1031
- )
1032
- decoder: Optional[JsonDecoder] = Field(
1033
- None,
1034
- description='Component decoding the response so records can be extracted.',
1035
- title='Decoder',
1036
- )
1037
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1508
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1038
1509
 
1039
1510
 
1040
1511
  class SessionTokenRequestApiKeyAuthenticator(BaseModel):
1041
- type: Literal['ApiKey']
1512
+ type: Literal["ApiKey"]
1042
1513
  inject_into: RequestOption = Field(
1043
1514
  ...,
1044
- description='Configure how the API Key will be sent in requests to the source API.',
1515
+ description="Configure how the API Key will be sent in requests to the source API.",
1045
1516
  examples=[
1046
- {'inject_into': 'header', 'field_name': 'Authorization'},
1047
- {'inject_into': 'request_parameter', 'field_name': 'authKey'},
1517
+ {"inject_into": "header", "field_name": "Authorization"},
1518
+ {"inject_into": "request_parameter", "field_name": "authKey"},
1048
1519
  ],
1049
- title='Inject API Key Into Outgoing HTTP Request',
1520
+ title="Inject API Key Into Outgoing HTTP Request",
1050
1521
  )
1051
1522
 
1052
1523
 
1053
1524
  class ListPartitionRouter(BaseModel):
1054
- type: Literal['ListPartitionRouter']
1525
+ type: Literal["ListPartitionRouter"]
1055
1526
  cursor_field: str = Field(
1056
1527
  ...,
1057
1528
  description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.',
1058
- examples=['section', "{{ config['section_key'] }}"],
1059
- title='Current Partition Value Identifier',
1529
+ examples=["section", "{{ config['section_key'] }}"],
1530
+ title="Current Partition Value Identifier",
1060
1531
  )
1061
1532
  values: Union[str, List[str]] = Field(
1062
1533
  ...,
1063
- description='The list of attributes being iterated over and used as input for the requests made to the source API.',
1064
- examples=[['section_a', 'section_b', 'section_c'], "{{ config['sections'] }}"],
1065
- title='Partition Values',
1534
+ description="The list of attributes being iterated over and used as input for the requests made to the source API.",
1535
+ examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"],
1536
+ title="Partition Values",
1066
1537
  )
1067
1538
  request_option: Optional[RequestOption] = Field(
1068
1539
  None,
1069
- description='A request option describing where the list value should be injected into and under what field name if applicable.',
1070
- title='Inject Partition Value Into Outgoing HTTP Request',
1540
+ description="A request option describing where the list value should be injected into and under what field name if applicable.",
1541
+ title="Inject Partition Value Into Outgoing HTTP Request",
1071
1542
  )
1072
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1543
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1073
1544
 
1074
1545
 
1075
1546
  class RecordSelector(BaseModel):
1076
- type: Literal['RecordSelector']
1547
+ type: Literal["RecordSelector"]
1077
1548
  extractor: Union[CustomRecordExtractor, DpathExtractor]
1078
1549
  record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field(
1079
1550
  None,
1080
- description='Responsible for filtering records to be emitted by the Source.',
1081
- title='Record Filter',
1551
+ description="Responsible for filtering records to be emitted by the Source.",
1552
+ title="Record Filter",
1082
1553
  )
1083
- schema_normalization: Optional[SchemaNormalization] = SchemaNormalization.None_
1084
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1554
+ schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field(
1555
+ SchemaNormalization.None_,
1556
+ description="Responsible for normalization according to the schema.",
1557
+ title="Schema Normalization",
1558
+ )
1559
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1560
+
1561
+
1562
+ class GzipParser(BaseModel):
1563
+ type: Literal["GzipParser"]
1564
+ inner_parser: Union[JsonLineParser, CsvParser]
1085
1565
 
1086
1566
 
1087
1567
  class Spec(BaseModel):
1088
- type: Literal['Spec']
1568
+ type: Literal["Spec"]
1089
1569
  connection_specification: Dict[str, Any] = Field(
1090
1570
  ...,
1091
- description='A connection specification describing how a the connector can be configured.',
1092
- title='Connection Specification',
1571
+ description="A connection specification describing how a the connector can be configured.",
1572
+ title="Connection Specification",
1093
1573
  )
1094
1574
  documentation_url: Optional[str] = Field(
1095
1575
  None,
1096
1576
  description="URL of the connector's documentation page.",
1097
- examples=['https://docs.airbyte.com/integrations/sources/dremio'],
1098
- title='Documentation URL',
1577
+ examples=["https://docs.airbyte.com/integrations/sources/dremio"],
1578
+ title="Documentation URL",
1099
1579
  )
1100
1580
  advanced_auth: Optional[AuthFlow] = Field(
1101
1581
  None,
1102
- description='Advanced specification for configuring the authentication flow.',
1103
- title='Advanced Auth',
1582
+ description="Advanced specification for configuring the authentication flow.",
1583
+ title="Advanced Auth",
1104
1584
  )
1105
1585
 
1106
1586
 
1107
1587
  class CompositeErrorHandler(BaseModel):
1108
- type: Literal['CompositeErrorHandler']
1588
+ type: Literal["CompositeErrorHandler"]
1109
1589
  error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field(
1110
1590
  ...,
1111
- description='List of error handlers to iterate on to determine how to handle a failed response.',
1112
- title='Error Handlers',
1591
+ description="List of error handlers to iterate on to determine how to handle a failed response.",
1592
+ title="Error Handlers",
1113
1593
  )
1114
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1594
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1115
1595
 
1116
1596
 
1117
- class DeclarativeSource(BaseModel):
1597
+ class CompositeRawDecoder(BaseModel):
1598
+ type: Literal["CompositeRawDecoder"]
1599
+ parser: Union[GzipParser, JsonLineParser, CsvParser]
1600
+
1601
+
1602
+ class DeclarativeSource1(BaseModel):
1118
1603
  class Config:
1119
1604
  extra = Extra.forbid
1120
1605
 
1121
- type: Literal['DeclarativeSource']
1606
+ type: Literal["DeclarativeSource"]
1122
1607
  check: CheckStream
1123
1608
  streams: List[DeclarativeStream]
1124
- version: str
1609
+ dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None
1610
+ version: str = Field(
1611
+ ...,
1612
+ description="The version of the Airbyte CDK used to build and test the source.",
1613
+ )
1614
+ schemas: Optional[Schemas] = None
1615
+ definitions: Optional[Dict[str, Any]] = None
1616
+ spec: Optional[Spec] = None
1617
+ concurrency_level: Optional[ConcurrencyLevel] = None
1618
+ metadata: Optional[Dict[str, Any]] = Field(
1619
+ None,
1620
+ description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
1621
+ )
1622
+ description: Optional[str] = Field(
1623
+ None,
1624
+ description="A description of the connector. It will be presented on the Source documentation page.",
1625
+ )
1626
+
1627
+
1628
+ class DeclarativeSource2(BaseModel):
1629
+ class Config:
1630
+ extra = Extra.forbid
1631
+
1632
+ type: Literal["DeclarativeSource"]
1633
+ check: CheckStream
1634
+ streams: Optional[List[DeclarativeStream]] = None
1635
+ dynamic_streams: List[DynamicDeclarativeStream]
1636
+ version: str = Field(
1637
+ ...,
1638
+ description="The version of the Airbyte CDK used to build and test the source.",
1639
+ )
1125
1640
  schemas: Optional[Schemas] = None
1126
1641
  definitions: Optional[Dict[str, Any]] = None
1127
1642
  spec: Optional[Spec] = None
1643
+ concurrency_level: Optional[ConcurrencyLevel] = None
1128
1644
  metadata: Optional[Dict[str, Any]] = Field(
1129
1645
  None,
1130
- description='For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.',
1646
+ description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
1647
+ )
1648
+ description: Optional[str] = Field(
1649
+ None,
1650
+ description="A description of the connector. It will be presented on the Source documentation page.",
1651
+ )
1652
+
1653
+
1654
+ class DeclarativeSource(BaseModel):
1655
+ class Config:
1656
+ extra = Extra.forbid
1657
+
1658
+ __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field(
1659
+ ...,
1660
+ description="An API source that extracts data according to its declarative components.",
1661
+ title="DeclarativeSource",
1131
1662
  )
1132
1663
 
1133
1664
 
@@ -1135,12 +1666,12 @@ class SelectiveAuthenticator(BaseModel):
1135
1666
  class Config:
1136
1667
  extra = Extra.allow
1137
1668
 
1138
- type: Literal['SelectiveAuthenticator']
1669
+ type: Literal["SelectiveAuthenticator"]
1139
1670
  authenticator_selection_path: List[str] = Field(
1140
1671
  ...,
1141
- description='Path of the field in config with selected authenticator name',
1142
- examples=[['auth'], ['auth', 'type']],
1143
- title='Authenticator Selection Path',
1672
+ description="Path of the field in config with selected authenticator name",
1673
+ examples=[["auth"], ["auth", "type"]],
1674
+ title="Authenticator Selection Path",
1144
1675
  )
1145
1676
  authenticators: Dict[
1146
1677
  str,
@@ -1150,127 +1681,150 @@ class SelectiveAuthenticator(BaseModel):
1150
1681
  BearerAuthenticator,
1151
1682
  CustomAuthenticator,
1152
1683
  OAuthAuthenticator,
1684
+ JwtAuthenticator,
1153
1685
  NoAuth,
1154
1686
  SessionTokenAuthenticator,
1155
1687
  LegacySessionTokenAuthenticator,
1156
1688
  ],
1157
1689
  ] = Field(
1158
1690
  ...,
1159
- description='Authenticators to select from.',
1691
+ description="Authenticators to select from.",
1160
1692
  examples=[
1161
1693
  {
1162
- 'authenticators': {
1163
- 'token': '#/definitions/ApiKeyAuthenticator',
1164
- 'oauth': '#/definitions/OAuthAuthenticator',
1694
+ "authenticators": {
1695
+ "token": "#/definitions/ApiKeyAuthenticator",
1696
+ "oauth": "#/definitions/OAuthAuthenticator",
1697
+ "jwt": "#/definitions/JwtAuthenticator",
1165
1698
  }
1166
1699
  }
1167
1700
  ],
1168
- title='Authenticators',
1701
+ title="Authenticators",
1169
1702
  )
1170
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1703
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1171
1704
 
1172
1705
 
1173
1706
  class DeclarativeStream(BaseModel):
1174
1707
  class Config:
1175
1708
  extra = Extra.allow
1176
1709
 
1177
- type: Literal['DeclarativeStream']
1178
- retriever: Union[CustomRetriever, SimpleRetriever] = Field(
1710
+ type: Literal["DeclarativeStream"]
1711
+ retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
1179
1712
  ...,
1180
- description='Component used to coordinate how records are extracted across stream slices and request pages.',
1181
- title='Retriever',
1713
+ description="Component used to coordinate how records are extracted across stream slices and request pages.",
1714
+ title="Retriever",
1182
1715
  )
1183
- incremental_sync: Optional[
1184
- Union[CustomIncrementalSync, DatetimeBasedCursor]
1185
- ] = Field(
1716
+ incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field(
1186
1717
  None,
1187
- description='Component used to fetch data incrementally based on a time field in the data.',
1188
- title='Incremental Sync',
1189
- )
1190
- name: Optional[str] = Field(
1191
- '', description='The stream name.', example=['Users'], title='Name'
1718
+ description="Component used to fetch data incrementally based on a time field in the data.",
1719
+ title="Incremental Sync",
1192
1720
  )
1721
+ name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name")
1193
1722
  primary_key: Optional[PrimaryKey] = Field(
1194
- '', description='The primary key of the stream.', title='Primary Key'
1723
+ "", description="The primary key of the stream.", title="Primary Key"
1195
1724
  )
1196
1725
  schema_loader: Optional[
1197
- Union[InlineSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]
1726
+ Union[
1727
+ DynamicSchemaLoader,
1728
+ InlineSchemaLoader,
1729
+ JsonFileSchemaLoader,
1730
+ CustomSchemaLoader,
1731
+ ]
1198
1732
  ] = Field(
1199
1733
  None,
1200
- description='Component used to retrieve the schema for the current stream.',
1201
- title='Schema Loader',
1734
+ description="Component used to retrieve the schema for the current stream.",
1735
+ title="Schema Loader",
1202
1736
  )
1203
1737
  transformations: Optional[
1204
- List[Union[AddFields, CustomTransformation, RemoveFields]]
1738
+ List[
1739
+ Union[
1740
+ AddFields,
1741
+ CustomTransformation,
1742
+ RemoveFields,
1743
+ KeysToLower,
1744
+ KeysToSnakeCase,
1745
+ FlattenFields,
1746
+ KeysReplace,
1747
+ ]
1748
+ ]
1205
1749
  ] = Field(
1206
1750
  None,
1207
- description='A list of transformations to be applied to each output record.',
1208
- title='Transformations',
1751
+ description="A list of transformations to be applied to each output record.",
1752
+ title="Transformations",
1209
1753
  )
1210
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1754
+ state_migrations: Optional[
1755
+ List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]
1756
+ ] = Field(
1757
+ [],
1758
+ description="Array of state migrations to be applied on the input state",
1759
+ title="State Migrations",
1760
+ )
1761
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1211
1762
 
1212
1763
 
1213
1764
  class SessionTokenAuthenticator(BaseModel):
1214
- type: Literal['SessionTokenAuthenticator']
1765
+ type: Literal["SessionTokenAuthenticator"]
1215
1766
  login_requester: HttpRequester = Field(
1216
1767
  ...,
1217
- description='Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.',
1768
+ description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.",
1218
1769
  examples=[
1219
1770
  {
1220
- 'type': 'HttpRequester',
1221
- 'url_base': 'https://my_api.com',
1222
- 'path': '/login',
1223
- 'authenticator': {
1224
- 'type': 'BasicHttpAuthenticator',
1225
- 'username': '{{ config.username }}',
1226
- 'password': '{{ config.password }}',
1771
+ "type": "HttpRequester",
1772
+ "url_base": "https://my_api.com",
1773
+ "path": "/login",
1774
+ "authenticator": {
1775
+ "type": "BasicHttpAuthenticator",
1776
+ "username": "{{ config.username }}",
1777
+ "password": "{{ config.password }}",
1227
1778
  },
1228
1779
  }
1229
1780
  ],
1230
- title='Login Requester',
1781
+ title="Login Requester",
1231
1782
  )
1232
1783
  session_token_path: List[str] = Field(
1233
1784
  ...,
1234
- description='The path in the response body returned from the login requester to the session token.',
1235
- examples=[['access_token'], ['result', 'token']],
1236
- title='Session Token Path',
1785
+ description="The path in the response body returned from the login requester to the session token.",
1786
+ examples=[["access_token"], ["result", "token"]],
1787
+ title="Session Token Path",
1237
1788
  )
1238
1789
  expiration_duration: Optional[str] = Field(
1239
1790
  None,
1240
- description='The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.',
1241
- examples=['PT1H', 'P1D'],
1242
- title='Expiration Duration',
1791
+ description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.",
1792
+ examples=["PT1H", "P1D"],
1793
+ title="Expiration Duration",
1243
1794
  )
1244
1795
  request_authentication: Union[
1245
1796
  SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator
1246
1797
  ] = Field(
1247
1798
  ...,
1248
- description='Authentication method to use for requests sent to the API, specifying how to inject the session token.',
1249
- title='Data Request Authentication',
1799
+ description="Authentication method to use for requests sent to the API, specifying how to inject the session token.",
1800
+ title="Data Request Authentication",
1801
+ )
1802
+ decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field(
1803
+ None, description="Component used to decode the response.", title="Decoder"
1250
1804
  )
1251
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1805
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1252
1806
 
1253
1807
 
1254
1808
  class HttpRequester(BaseModel):
1255
- type: Literal['HttpRequester']
1809
+ type: Literal["HttpRequester"]
1256
1810
  url_base: str = Field(
1257
1811
  ...,
1258
- description='Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.',
1812
+ description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
1259
1813
  examples=[
1260
- 'https://connect.squareup.com/v2',
1814
+ "https://connect.squareup.com/v2",
1261
1815
  "{{ config['base_url'] or 'https://app.posthog.com'}}/api/",
1262
1816
  ],
1263
- title='API Base URL',
1817
+ title="API Base URL",
1264
1818
  )
1265
1819
  path: str = Field(
1266
1820
  ...,
1267
- description='Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.',
1821
+ description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
1268
1822
  examples=[
1269
- '/products',
1823
+ "/products",
1270
1824
  "/quotes/{{ stream_partition['id'] }}/quote_line_groups",
1271
1825
  "/trades/{{ config['symbol_id'] }}/history",
1272
1826
  ],
1273
- title='URL Path',
1827
+ title="URL Path",
1274
1828
  )
1275
1829
  authenticator: Optional[
1276
1830
  Union[
@@ -1279,6 +1833,7 @@ class HttpRequester(BaseModel):
1279
1833
  BearerAuthenticator,
1280
1834
  CustomAuthenticator,
1281
1835
  OAuthAuthenticator,
1836
+ JwtAuthenticator,
1282
1837
  NoAuth,
1283
1838
  SessionTokenAuthenticator,
1284
1839
  LegacySessionTokenAuthenticator,
@@ -1286,101 +1841,139 @@ class HttpRequester(BaseModel):
1286
1841
  ]
1287
1842
  ] = Field(
1288
1843
  None,
1289
- description='Authentication method to use for requests sent to the API.',
1290
- title='Authenticator',
1844
+ description="Authentication method to use for requests sent to the API.",
1845
+ title="Authenticator",
1291
1846
  )
1292
1847
  error_handler: Optional[
1293
1848
  Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]
1294
1849
  ] = Field(
1295
1850
  None,
1296
- description='Error handler component that defines how to handle errors.',
1297
- title='Error Handler',
1851
+ description="Error handler component that defines how to handle errors.",
1852
+ title="Error Handler",
1298
1853
  )
1299
1854
  http_method: Optional[HttpMethod] = Field(
1300
1855
  HttpMethod.GET,
1301
- description='The HTTP method used to fetch data from the source (can be GET or POST).',
1302
- examples=['GET', 'POST'],
1303
- title='HTTP Method',
1856
+ description="The HTTP method used to fetch data from the source (can be GET or POST).",
1857
+ examples=["GET", "POST"],
1858
+ title="HTTP Method",
1304
1859
  )
1305
1860
  request_body_data: Optional[Union[str, Dict[str, str]]] = Field(
1306
1861
  None,
1307
- description='Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.',
1862
+ description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.",
1308
1863
  examples=[
1309
1864
  '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n'
1310
1865
  ],
1311
- title='Request Body Payload (Non-JSON)',
1866
+ title="Request Body Payload (Non-JSON)",
1312
1867
  )
1313
1868
  request_body_json: Optional[Union[str, Dict[str, Any]]] = Field(
1314
1869
  None,
1315
- description='Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.',
1870
+ description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.",
1316
1871
  examples=[
1317
- {'sort_order': 'ASC', 'sort_field': 'CREATED_AT'},
1318
- {'key': "{{ config['value'] }}"},
1319
- {'sort': {'field': 'updated_at', 'order': 'ascending'}},
1872
+ {"sort_order": "ASC", "sort_field": "CREATED_AT"},
1873
+ {"key": "{{ config['value'] }}"},
1874
+ {"sort": {"field": "updated_at", "order": "ascending"}},
1320
1875
  ],
1321
- title='Request Body JSON Payload',
1876
+ title="Request Body JSON Payload",
1322
1877
  )
1323
1878
  request_headers: Optional[Union[str, Dict[str, str]]] = Field(
1324
1879
  None,
1325
- description='Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.',
1326
- examples=[{'Output-Format': 'JSON'}, {'Version': "{{ config['version'] }}"}],
1327
- title='Request Headers',
1880
+ description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.",
1881
+ examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
1882
+ title="Request Headers",
1328
1883
  )
1329
1884
  request_parameters: Optional[Union[str, Dict[str, str]]] = Field(
1330
1885
  None,
1331
- description='Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.',
1886
+ description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
1332
1887
  examples=[
1333
- {'unit': 'day'},
1888
+ {"unit": "day"},
1334
1889
  {
1335
- 'query': 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
1890
+ "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
1336
1891
  },
1337
- {'searchIn': "{{ ','.join(config.get('search_in', [])) }}"},
1338
- {'sort_by[asc]': 'updated_at'},
1892
+ {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
1893
+ {"sort_by[asc]": "updated_at"},
1339
1894
  ],
1340
- title='Query Parameters',
1895
+ title="Query Parameters",
1341
1896
  )
1342
1897
  use_cache: Optional[bool] = Field(
1343
1898
  False,
1344
- description='Enables stream requests caching. This field is automatically set by the CDK.',
1345
- title='Use Cache',
1899
+ description="Enables stream requests caching. This field is automatically set by the CDK.",
1900
+ title="Use Cache",
1901
+ )
1902
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1903
+
1904
+
1905
+ class DynamicSchemaLoader(BaseModel):
1906
+ type: Literal["DynamicSchemaLoader"]
1907
+ retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
1908
+ ...,
1909
+ description="Component used to coordinate how records are extracted across stream slices and request pages.",
1910
+ title="Retriever",
1911
+ )
1912
+ schema_transformations: Optional[
1913
+ List[
1914
+ Union[
1915
+ AddFields,
1916
+ CustomTransformation,
1917
+ RemoveFields,
1918
+ KeysToLower,
1919
+ KeysToSnakeCase,
1920
+ FlattenFields,
1921
+ KeysReplace,
1922
+ ]
1923
+ ]
1924
+ ] = Field(
1925
+ None,
1926
+ description="A list of transformations to be applied to the schema.",
1927
+ title="Schema Transformations",
1346
1928
  )
1347
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1929
+ schema_type_identifier: SchemaTypeIdentifier
1930
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1348
1931
 
1349
1932
 
1350
1933
  class ParentStreamConfig(BaseModel):
1351
- type: Literal['ParentStreamConfig']
1934
+ type: Literal["ParentStreamConfig"]
1352
1935
  parent_key: str = Field(
1353
1936
  ...,
1354
- description='The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.',
1355
- examples=['id', "{{ config['parent_record_id'] }}"],
1356
- title='Parent Key',
1937
+ description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.",
1938
+ examples=["id", "{{ config['parent_record_id'] }}"],
1939
+ title="Parent Key",
1357
1940
  )
1358
1941
  stream: DeclarativeStream = Field(
1359
- ..., description='Reference to the parent stream.', title='Parent Stream'
1942
+ ..., description="Reference to the parent stream.", title="Parent Stream"
1360
1943
  )
1361
1944
  partition_field: str = Field(
1362
1945
  ...,
1363
- description='While iterating over parent records during a sync, the parent_key value can be referenced by using this field.',
1364
- examples=['parent_id', "{{ config['parent_partition_field'] }}"],
1365
- title='Current Parent Key Value Identifier',
1946
+ description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.",
1947
+ examples=["parent_id", "{{ config['parent_partition_field'] }}"],
1948
+ title="Current Parent Key Value Identifier",
1366
1949
  )
1367
1950
  request_option: Optional[RequestOption] = Field(
1368
1951
  None,
1369
- description='A request option describing where the parent key value should be injected into and under what field name if applicable.',
1370
- title='Request Option',
1952
+ description="A request option describing where the parent key value should be injected into and under what field name if applicable.",
1953
+ title="Request Option",
1954
+ )
1955
+ incremental_dependency: Optional[bool] = Field(
1956
+ False,
1957
+ description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.",
1958
+ title="Incremental Dependency",
1959
+ )
1960
+ extra_fields: Optional[List[List[str]]] = Field(
1961
+ None,
1962
+ description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.",
1963
+ title="Extra Fields",
1371
1964
  )
1372
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1965
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1373
1966
 
1374
1967
 
1375
1968
  class SimpleRetriever(BaseModel):
1376
- type: Literal['SimpleRetriever']
1969
+ type: Literal["SimpleRetriever"]
1377
1970
  record_selector: RecordSelector = Field(
1378
1971
  ...,
1379
- description='Component that describes how to extract records from a HTTP response.',
1972
+ description="Component that describes how to extract records from a HTTP response.",
1380
1973
  )
1381
1974
  requester: Union[CustomRequester, HttpRequester] = Field(
1382
1975
  ...,
1383
- description='Requester component that describes how to prepare HTTP requests to send to the source API.',
1976
+ description="Requester component that describes how to prepare HTTP requests to send to the source API.",
1384
1977
  )
1385
1978
  paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
1386
1979
  None,
@@ -1388,40 +1981,163 @@ class SimpleRetriever(BaseModel):
1388
1981
  )
1389
1982
  ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field(
1390
1983
  False,
1391
- description='If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.',
1984
+ description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.",
1392
1985
  )
1393
1986
  partition_router: Optional[
1394
1987
  Union[
1395
1988
  CustomPartitionRouter,
1396
1989
  ListPartitionRouter,
1397
1990
  SubstreamPartitionRouter,
1398
- List[
1399
- Union[
1400
- CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
1401
- ]
1402
- ],
1991
+ List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
1403
1992
  ]
1404
1993
  ] = Field(
1405
1994
  [],
1406
- description='PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.',
1407
- title='Partition Router',
1995
+ description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
1996
+ title="Partition Router",
1408
1997
  )
1409
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1998
+ decoder: Optional[
1999
+ Union[
2000
+ CustomDecoder,
2001
+ JsonDecoder,
2002
+ JsonlDecoder,
2003
+ IterableDecoder,
2004
+ XmlDecoder,
2005
+ GzipJsonDecoder,
2006
+ CompositeRawDecoder,
2007
+ ]
2008
+ ] = Field(
2009
+ None,
2010
+ description="Component decoding the response so records can be extracted.",
2011
+ title="Decoder",
2012
+ )
2013
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
2014
+
2015
+
2016
+ class AsyncRetriever(BaseModel):
2017
+ type: Literal["AsyncRetriever"]
2018
+ record_selector: RecordSelector = Field(
2019
+ ...,
2020
+ description="Component that describes how to extract records from a HTTP response.",
2021
+ )
2022
+ status_mapping: AsyncJobStatusMap = Field(
2023
+ ..., description="Async Job Status to Airbyte CDK Async Job Status mapping."
2024
+ )
2025
+ status_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
2026
+ ..., description="Responsible for fetching the actual status of the async job."
2027
+ )
2028
+ urls_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
2029
+ ...,
2030
+ description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.",
2031
+ )
2032
+ download_extractor: Optional[
2033
+ Union[CustomRecordExtractor, DpathExtractor, ResponseToFileExtractor]
2034
+ ] = Field(None, description="Responsible for fetching the records from provided urls.")
2035
+ creation_requester: Union[CustomRequester, HttpRequester] = Field(
2036
+ ...,
2037
+ description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.",
2038
+ )
2039
+ polling_requester: Union[CustomRequester, HttpRequester] = Field(
2040
+ ...,
2041
+ description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.",
2042
+ )
2043
+ download_requester: Union[CustomRequester, HttpRequester] = Field(
2044
+ ...,
2045
+ description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.",
2046
+ )
2047
+ download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
2048
+ None,
2049
+ description="Paginator component that describes how to navigate through the API's pages during download.",
2050
+ )
2051
+ abort_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
2052
+ None,
2053
+ description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.",
2054
+ )
2055
+ delete_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
2056
+ None,
2057
+ description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.",
2058
+ )
2059
+ partition_router: Optional[
2060
+ Union[
2061
+ CustomPartitionRouter,
2062
+ ListPartitionRouter,
2063
+ SubstreamPartitionRouter,
2064
+ List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
2065
+ ]
2066
+ ] = Field(
2067
+ [],
2068
+ description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
2069
+ title="Partition Router",
2070
+ )
2071
+ decoder: Optional[
2072
+ Union[
2073
+ CustomDecoder,
2074
+ JsonDecoder,
2075
+ JsonlDecoder,
2076
+ IterableDecoder,
2077
+ XmlDecoder,
2078
+ GzipJsonDecoder,
2079
+ ]
2080
+ ] = Field(
2081
+ None,
2082
+ description="Component decoding the response so records can be extracted.",
2083
+ title="Decoder",
2084
+ )
2085
+ download_decoder: Optional[
2086
+ Union[
2087
+ CustomDecoder,
2088
+ JsonDecoder,
2089
+ JsonlDecoder,
2090
+ IterableDecoder,
2091
+ XmlDecoder,
2092
+ GzipJsonDecoder,
2093
+ ]
2094
+ ] = Field(
2095
+ None,
2096
+ description="Component decoding the download response so records can be extracted.",
2097
+ title="Download Decoder",
2098
+ )
2099
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1410
2100
 
1411
2101
 
1412
2102
  class SubstreamPartitionRouter(BaseModel):
1413
- type: Literal['SubstreamPartitionRouter']
2103
+ type: Literal["SubstreamPartitionRouter"]
1414
2104
  parent_stream_configs: List[ParentStreamConfig] = Field(
1415
2105
  ...,
1416
- description='Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.',
1417
- title='Parent Stream Configs',
2106
+ description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.",
2107
+ title="Parent Stream Configs",
2108
+ )
2109
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
2110
+
2111
+
2112
+ class HttpComponentsResolver(BaseModel):
2113
+ type: Literal["HttpComponentsResolver"]
2114
+ retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
2115
+ ...,
2116
+ description="Component used to coordinate how records are extracted across stream slices and request pages.",
2117
+ title="Retriever",
2118
+ )
2119
+ components_mapping: List[ComponentMappingDefinition]
2120
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
2121
+
2122
+
2123
+ class DynamicDeclarativeStream(BaseModel):
2124
+ type: Literal["DynamicDeclarativeStream"]
2125
+ stream_template: DeclarativeStream = Field(
2126
+ ..., description="Reference to the stream template.", title="Stream Template"
2127
+ )
2128
+ components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field(
2129
+ ...,
2130
+ description="Component resolve and populates stream templates with components values.",
2131
+ title="Components Resolver",
1418
2132
  )
1419
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1420
2133
 
1421
2134
 
1422
2135
  CompositeErrorHandler.update_forward_refs()
1423
- DeclarativeSource.update_forward_refs()
2136
+ DeclarativeSource1.update_forward_refs()
2137
+ DeclarativeSource2.update_forward_refs()
1424
2138
  SelectiveAuthenticator.update_forward_refs()
1425
2139
  DeclarativeStream.update_forward_refs()
1426
2140
  SessionTokenAuthenticator.update_forward_refs()
2141
+ DynamicSchemaLoader.update_forward_refs()
1427
2142
  SimpleRetriever.update_forward_refs()
2143
+ AsyncRetriever.update_forward_refs()