airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (518) hide show
  1. airbyte_cdk/__init__.py +355 -6
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +29 -10
  7. airbyte_cdk/connector.py +24 -24
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
  10. airbyte_cdk/connector_builder/main.py +45 -13
  11. airbyte_cdk/connector_builder/message_grouper.py +189 -50
  12. airbyte_cdk/connector_builder/models.py +3 -2
  13. airbyte_cdk/destinations/__init__.py +4 -3
  14. airbyte_cdk/destinations/destination.py +54 -20
  15. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  16. airbyte_cdk/destinations/vector_db_based/config.py +40 -17
  17. airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
  18. airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
  19. airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
  20. airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
  21. airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
  22. airbyte_cdk/entrypoint.py +153 -44
  23. airbyte_cdk/exception_handler.py +21 -3
  24. airbyte_cdk/logger.py +30 -44
  25. airbyte_cdk/models/__init__.py +13 -2
  26. airbyte_cdk/models/airbyte_protocol.py +86 -1
  27. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  28. airbyte_cdk/models/file_transfer_record_message.py +13 -0
  29. airbyte_cdk/models/well_known_types.py +1 -1
  30. airbyte_cdk/sources/__init__.py +5 -1
  31. airbyte_cdk/sources/abstract_source.py +125 -79
  32. airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
  33. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
  34. airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
  35. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
  36. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  37. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
  38. airbyte_cdk/sources/config.py +3 -2
  39. airbyte_cdk/sources/connector_state_manager.py +49 -83
  40. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  41. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
  42. airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
  43. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  44. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  45. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  46. airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
  47. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
  48. airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
  49. airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
  50. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
  51. airbyte_cdk/sources/declarative/auth/token.py +28 -10
  52. airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
  53. airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
  54. airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
  55. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  56. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  57. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
  58. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
  59. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
  60. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1213 -88
  61. airbyte_cdk/sources/declarative/declarative_source.py +5 -2
  62. airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
  63. airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
  64. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
  65. airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
  66. airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
  67. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  68. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  69. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  70. airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
  71. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
  72. airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
  73. airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
  74. airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
  75. airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
  76. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
  77. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  78. airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
  79. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +346 -0
  80. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
  81. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  82. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
  83. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +173 -74
  84. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  85. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  86. airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
  87. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
  88. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
  89. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
  90. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
  91. airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
  92. airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
  93. airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
  94. airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
  95. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  96. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  97. airbyte_cdk/sources/declarative/models/__init__.py +1 -1
  98. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1329 -595
  99. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
  100. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
  101. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
  102. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1763 -226
  103. airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
  104. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  105. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  106. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
  107. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  108. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
  109. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
  110. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
  111. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
  112. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
  113. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
  114. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
  115. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
  116. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
  117. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
  118. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
  119. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
  120. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  121. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
  122. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
  123. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
  124. airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
  125. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
  126. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
  127. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
  128. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
  129. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
  130. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
  131. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
  132. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
  133. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
  134. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
  135. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
  136. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
  137. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  138. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
  139. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
  140. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
  141. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
  142. airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
  143. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  144. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  145. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  146. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  147. airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
  148. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
  149. airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
  150. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
  151. airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
  152. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
  153. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
  154. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
  155. airbyte_cdk/sources/declarative/spec/spec.py +12 -5
  156. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
  157. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
  158. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
  159. airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
  160. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  161. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  162. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  163. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  164. airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
  165. airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
  166. airbyte_cdk/sources/declarative/types.py +19 -110
  167. airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
  168. airbyte_cdk/sources/embedded/base_integration.py +16 -5
  169. airbyte_cdk/sources/embedded/catalog.py +16 -4
  170. airbyte_cdk/sources/embedded/runner.py +19 -3
  171. airbyte_cdk/sources/embedded/tools.py +5 -2
  172. airbyte_cdk/sources/file_based/README.md +152 -0
  173. airbyte_cdk/sources/file_based/__init__.py +24 -0
  174. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
  175. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
  176. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
  177. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
  178. airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
  179. airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
  180. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  181. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
  182. airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
  183. airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
  184. airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
  185. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
  186. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
  187. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
  188. airbyte_cdk/sources/file_based/exceptions.py +18 -15
  189. airbyte_cdk/sources/file_based/file_based_source.py +140 -33
  190. airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
  191. airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
  192. airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
  193. airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
  194. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  195. airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
  196. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
  197. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
  198. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
  199. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
  200. airbyte_cdk/sources/file_based/remote_file.py +1 -1
  201. airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
  202. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
  203. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
  204. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
  205. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
  206. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
  207. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
  208. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
  209. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
  210. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
  211. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
  212. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
  213. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
  214. airbyte_cdk/sources/http_logger.py +8 -3
  215. airbyte_cdk/sources/message/__init__.py +7 -1
  216. airbyte_cdk/sources/message/repository.py +18 -4
  217. airbyte_cdk/sources/source.py +42 -38
  218. airbyte_cdk/sources/streams/__init__.py +2 -2
  219. airbyte_cdk/sources/streams/availability_strategy.py +54 -3
  220. airbyte_cdk/sources/streams/call_rate.py +64 -21
  221. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  222. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  223. airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
  224. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  225. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  226. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  227. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  228. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
  229. airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
  230. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
  231. airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
  232. airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
  233. airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
  234. airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
  235. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
  236. airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
  237. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
  238. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  239. airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
  240. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
  241. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
  242. airbyte_cdk/sources/streams/core.py +412 -87
  243. airbyte_cdk/sources/streams/http/__init__.py +2 -1
  244. airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
  245. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  246. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  247. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  248. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  249. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  250. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  251. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  252. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  253. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  254. airbyte_cdk/sources/streams/http/exceptions.py +27 -7
  255. airbyte_cdk/sources/streams/http/http.py +369 -246
  256. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  257. airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
  258. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
  259. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
  260. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
  261. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
  262. airbyte_cdk/sources/types.py +154 -0
  263. airbyte_cdk/sources/utils/record_helper.py +36 -21
  264. airbyte_cdk/sources/utils/schema_helpers.py +13 -6
  265. airbyte_cdk/sources/utils/slice_logger.py +4 -1
  266. airbyte_cdk/sources/utils/transform.py +54 -20
  267. airbyte_cdk/sql/_util/hashing.py +34 -0
  268. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  269. airbyte_cdk/sql/constants.py +32 -0
  270. airbyte_cdk/sql/exceptions.py +235 -0
  271. airbyte_cdk/sql/secrets.py +123 -0
  272. airbyte_cdk/sql/shared/__init__.py +15 -0
  273. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  274. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  275. airbyte_cdk/sql/types.py +160 -0
  276. airbyte_cdk/test/catalog_builder.py +70 -18
  277. airbyte_cdk/test/entrypoint_wrapper.py +117 -42
  278. airbyte_cdk/test/mock_http/__init__.py +1 -1
  279. airbyte_cdk/test/mock_http/matcher.py +6 -0
  280. airbyte_cdk/test/mock_http/mocker.py +57 -10
  281. airbyte_cdk/test/mock_http/request.py +19 -3
  282. airbyte_cdk/test/mock_http/response.py +3 -1
  283. airbyte_cdk/test/mock_http/response_builder.py +32 -16
  284. airbyte_cdk/test/state_builder.py +18 -10
  285. airbyte_cdk/test/utils/__init__.py +1 -0
  286. airbyte_cdk/test/utils/data.py +24 -0
  287. airbyte_cdk/test/utils/http_mocking.py +16 -0
  288. airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
  289. airbyte_cdk/test/utils/reading.py +26 -0
  290. airbyte_cdk/utils/__init__.py +2 -1
  291. airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
  292. airbyte_cdk/utils/analytics_message.py +10 -2
  293. airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
  294. airbyte_cdk/utils/event_timing.py +10 -10
  295. airbyte_cdk/utils/mapping_helpers.py +3 -1
  296. airbyte_cdk/utils/message_utils.py +20 -11
  297. airbyte_cdk/utils/print_buffer.py +75 -0
  298. airbyte_cdk/utils/schema_inferrer.py +198 -28
  299. airbyte_cdk/utils/slice_hasher.py +30 -0
  300. airbyte_cdk/utils/spec_schema_transformations.py +6 -3
  301. airbyte_cdk/utils/stream_status_utils.py +8 -1
  302. airbyte_cdk/utils/traced_exception.py +61 -21
  303. airbyte_cdk-6.17.1.dev0.dist-info/METADATA +109 -0
  304. airbyte_cdk-6.17.1.dev0.dist-info/RECORD +350 -0
  305. {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/WHEEL +1 -2
  306. airbyte_cdk-6.17.1.dev0.dist-info/entry_points.txt +3 -0
  307. airbyte_cdk/sources/declarative/create_partial.py +0 -92
  308. airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
  309. airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
  310. airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
  311. airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
  312. airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
  313. airbyte_cdk/sources/deprecated/base_source.py +0 -94
  314. airbyte_cdk/sources/deprecated/client.py +0 -99
  315. airbyte_cdk/sources/singer/__init__.py +0 -8
  316. airbyte_cdk/sources/singer/singer_helpers.py +0 -304
  317. airbyte_cdk/sources/singer/source.py +0 -186
  318. airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
  319. airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
  320. airbyte_cdk/sources/streams/http/auth/core.py +0 -29
  321. airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
  322. airbyte_cdk/sources/streams/http/auth/token.py +0 -47
  323. airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
  324. airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
  325. airbyte_cdk/sources/utils/schema_models.py +0 -84
  326. airbyte_cdk-0.72.0.dist-info/METADATA +0 -243
  327. airbyte_cdk-0.72.0.dist-info/RECORD +0 -466
  328. airbyte_cdk-0.72.0.dist-info/top_level.txt +0 -3
  329. source_declarative_manifest/main.py +0 -29
  330. unit_tests/connector_builder/__init__.py +0 -3
  331. unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
  332. unit_tests/connector_builder/test_message_grouper.py +0 -713
  333. unit_tests/connector_builder/utils.py +0 -27
  334. unit_tests/destinations/test_destination.py +0 -243
  335. unit_tests/singer/test_singer_helpers.py +0 -56
  336. unit_tests/singer/test_singer_source.py +0 -112
  337. unit_tests/sources/__init__.py +0 -0
  338. unit_tests/sources/concurrent_source/__init__.py +0 -3
  339. unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
  340. unit_tests/sources/declarative/__init__.py +0 -3
  341. unit_tests/sources/declarative/auth/__init__.py +0 -3
  342. unit_tests/sources/declarative/auth/test_oauth.py +0 -331
  343. unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
  344. unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
  345. unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
  346. unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
  347. unit_tests/sources/declarative/checks/__init__.py +0 -3
  348. unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
  349. unit_tests/sources/declarative/decoders/__init__.py +0 -0
  350. unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
  351. unit_tests/sources/declarative/external_component.py +0 -13
  352. unit_tests/sources/declarative/extractors/__init__.py +0 -3
  353. unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
  354. unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
  355. unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
  356. unit_tests/sources/declarative/incremental/__init__.py +0 -0
  357. unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
  358. unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
  359. unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
  360. unit_tests/sources/declarative/interpolation/__init__.py +0 -3
  361. unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
  362. unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
  363. unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
  364. unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
  365. unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
  366. unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
  367. unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
  368. unit_tests/sources/declarative/parsers/__init__.py +0 -3
  369. unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
  370. unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
  371. unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1841
  372. unit_tests/sources/declarative/parsers/testing_components.py +0 -36
  373. unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
  374. unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
  375. unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
  376. unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
  377. unit_tests/sources/declarative/requesters/__init__.py +0 -3
  378. unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
  379. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
  380. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
  381. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
  382. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
  383. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
  384. unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
  385. unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
  386. unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
  387. unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
  388. unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
  389. unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
  390. unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
  391. unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
  392. unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
  393. unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
  394. unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
  395. unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
  396. unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
  397. unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
  398. unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
  399. unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
  400. unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
  401. unit_tests/sources/declarative/retrievers/__init__.py +0 -3
  402. unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
  403. unit_tests/sources/declarative/schema/__init__.py +0 -6
  404. unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
  405. unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
  406. unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
  407. unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
  408. unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
  409. unit_tests/sources/declarative/states/__init__.py +0 -3
  410. unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
  411. unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
  412. unit_tests/sources/declarative/test_create_partial.py +0 -83
  413. unit_tests/sources/declarative/test_declarative_stream.py +0 -103
  414. unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
  415. unit_tests/sources/declarative/test_types.py +0 -39
  416. unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
  417. unit_tests/sources/file_based/__init__.py +0 -0
  418. unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
  419. unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
  420. unit_tests/sources/file_based/config/__init__.py +0 -0
  421. unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
  422. unit_tests/sources/file_based/config/test_csv_format.py +0 -34
  423. unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
  424. unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
  425. unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
  426. unit_tests/sources/file_based/file_types/__init__.py +0 -0
  427. unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
  428. unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
  429. unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
  430. unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
  431. unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
  432. unit_tests/sources/file_based/helpers.py +0 -70
  433. unit_tests/sources/file_based/in_memory_files_source.py +0 -211
  434. unit_tests/sources/file_based/scenarios/__init__.py +0 -0
  435. unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
  436. unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
  437. unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
  438. unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
  439. unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
  440. unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
  441. unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
  442. unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
  443. unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
  444. unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
  445. unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
  446. unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
  447. unit_tests/sources/file_based/stream/__init__.py +0 -0
  448. unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
  449. unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
  450. unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
  451. unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
  452. unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
  453. unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
  454. unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
  455. unit_tests/sources/file_based/test_scenarios.py +0 -253
  456. unit_tests/sources/file_based/test_schema_helpers.py +0 -346
  457. unit_tests/sources/fixtures/__init__.py +0 -3
  458. unit_tests/sources/fixtures/source_test_fixture.py +0 -153
  459. unit_tests/sources/message/__init__.py +0 -0
  460. unit_tests/sources/message/test_repository.py +0 -153
  461. unit_tests/sources/streams/__init__.py +0 -0
  462. unit_tests/sources/streams/concurrent/__init__.py +0 -3
  463. unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
  464. unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
  465. unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
  466. unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
  467. unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
  468. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
  469. unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
  470. unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
  471. unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
  472. unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
  473. unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
  474. unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
  475. unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
  476. unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
  477. unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
  478. unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
  479. unit_tests/sources/streams/http/__init__.py +0 -0
  480. unit_tests/sources/streams/http/auth/__init__.py +0 -0
  481. unit_tests/sources/streams/http/auth/test_auth.py +0 -173
  482. unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
  483. unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
  484. unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
  485. unit_tests/sources/streams/http/test_http.py +0 -635
  486. unit_tests/sources/streams/test_availability_strategy.py +0 -70
  487. unit_tests/sources/streams/test_call_rate.py +0 -300
  488. unit_tests/sources/streams/test_stream_read.py +0 -405
  489. unit_tests/sources/streams/test_streams_core.py +0 -184
  490. unit_tests/sources/test_abstract_source.py +0 -1442
  491. unit_tests/sources/test_concurrent_source.py +0 -112
  492. unit_tests/sources/test_config.py +0 -92
  493. unit_tests/sources/test_connector_state_manager.py +0 -482
  494. unit_tests/sources/test_http_logger.py +0 -252
  495. unit_tests/sources/test_integration_source.py +0 -86
  496. unit_tests/sources/test_source.py +0 -684
  497. unit_tests/sources/test_source_read.py +0 -460
  498. unit_tests/test/__init__.py +0 -0
  499. unit_tests/test/mock_http/__init__.py +0 -0
  500. unit_tests/test/mock_http/test_matcher.py +0 -53
  501. unit_tests/test/mock_http/test_mocker.py +0 -214
  502. unit_tests/test/mock_http/test_request.py +0 -117
  503. unit_tests/test/mock_http/test_response_builder.py +0 -177
  504. unit_tests/test/test_entrypoint_wrapper.py +0 -240
  505. unit_tests/utils/__init__.py +0 -0
  506. unit_tests/utils/test_datetime_format_inferrer.py +0 -60
  507. unit_tests/utils/test_mapping_helpers.py +0 -54
  508. unit_tests/utils/test_message_utils.py +0 -91
  509. unit_tests/utils/test_rate_limiting.py +0 -26
  510. unit_tests/utils/test_schema_inferrer.py +0 -202
  511. unit_tests/utils/test_secret_utils.py +0 -135
  512. unit_tests/utils/test_stream_status_utils.py +0 -61
  513. unit_tests/utils/test_traced_exception.py +0 -107
  514. /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
  515. {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
  516. {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
  517. {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
  518. {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/LICENSE.txt +0 -0
@@ -4,651 +4,1108 @@
4
4
  from __future__ import annotations
5
5
 
6
6
  from enum import Enum
7
- from typing import Any, Dict, List, Optional, Union
7
+ from typing import Any, Dict, List, Literal, Optional, Union
8
8
 
9
- from pydantic import BaseModel, Extra, Field
10
- from typing_extensions import Literal
9
+ from pydantic.v1 import BaseModel, Extra, Field
11
10
 
12
11
 
13
12
  class AuthFlowType(Enum):
14
- oauth2_0 = 'oauth2.0'
15
- oauth1_0 = 'oauth1.0'
13
+ oauth2_0 = "oauth2.0"
14
+ oauth1_0 = "oauth1.0"
16
15
 
17
16
 
18
17
  class BasicHttpAuthenticator(BaseModel):
19
- type: Literal['BasicHttpAuthenticator']
18
+ type: Literal["BasicHttpAuthenticator"]
20
19
  username: str = Field(
21
20
  ...,
22
- description='The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.',
21
+ description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.",
23
22
  examples=["{{ config['username'] }}", "{{ config['api_key'] }}"],
24
- title='Username',
23
+ title="Username",
25
24
  )
26
25
  password: Optional[str] = Field(
27
- '',
28
- description='The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.',
29
- examples=["{{ config['password'] }}", ''],
30
- title='Password',
26
+ "",
27
+ description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.",
28
+ examples=["{{ config['password'] }}", ""],
29
+ title="Password",
31
30
  )
32
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
31
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
33
32
 
34
33
 
35
34
  class BearerAuthenticator(BaseModel):
36
- type: Literal['BearerAuthenticator']
35
+ type: Literal["BearerAuthenticator"]
37
36
  api_token: str = Field(
38
37
  ...,
39
- description='Token to inject as request header for authenticating with the API.',
38
+ description="Token to inject as request header for authenticating with the API.",
40
39
  examples=["{{ config['api_key'] }}", "{{ config['token'] }}"],
41
- title='Bearer Token',
40
+ title="Bearer Token",
42
41
  )
43
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
42
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
44
43
 
45
44
 
46
45
  class CheckStream(BaseModel):
47
- type: Literal['CheckStream']
46
+ type: Literal["CheckStream"]
48
47
  stream_names: List[str] = Field(
49
48
  ...,
50
- description='Names of the streams to try reading from when running a check operation.',
51
- examples=[['users'], ['users', 'contacts']],
52
- title='Stream Names',
49
+ description="Names of the streams to try reading from when running a check operation.",
50
+ examples=[["users"], ["users", "contacts"]],
51
+ title="Stream Names",
53
52
  )
54
53
 
55
54
 
55
+ class ConcurrencyLevel(BaseModel):
56
+ type: Optional[Literal["ConcurrencyLevel"]] = None
57
+ default_concurrency: Union[int, str] = Field(
58
+ ...,
59
+ description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.",
60
+ examples=[10, "{{ config['num_workers'] or 10 }}"],
61
+ title="Default Concurrency",
62
+ )
63
+ max_concurrency: Optional[int] = Field(
64
+ None,
65
+ description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.",
66
+ examples=[20, 100],
67
+ title="Max Concurrency",
68
+ )
69
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
70
+
71
+
56
72
  class ConstantBackoffStrategy(BaseModel):
57
- type: Literal['ConstantBackoffStrategy']
73
+ type: Literal["ConstantBackoffStrategy"]
58
74
  backoff_time_in_seconds: Union[float, str] = Field(
59
75
  ...,
60
- description='Backoff time in seconds.',
76
+ description="Backoff time in seconds.",
61
77
  examples=[30, 30.5, "{{ config['backoff_time'] }}"],
62
- title='Backoff Time',
78
+ title="Backoff Time",
79
+ )
80
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
81
+
82
+
83
+ class CursorPagination(BaseModel):
84
+ type: Literal["CursorPagination"]
85
+ cursor_value: str = Field(
86
+ ...,
87
+ description="Value of the cursor defining the next page to fetch.",
88
+ examples=[
89
+ "{{ headers.link.next.cursor }}",
90
+ "{{ last_record['key'] }}",
91
+ "{{ response['nextPage'] }}",
92
+ ],
93
+ title="Cursor Value",
94
+ )
95
+ page_size: Optional[int] = Field(
96
+ None,
97
+ description="The number of records to include in each pages.",
98
+ examples=[100],
99
+ title="Page Size",
100
+ )
101
+ stop_condition: Optional[str] = Field(
102
+ None,
103
+ description="Template string evaluating when to stop paginating.",
104
+ examples=[
105
+ "{{ response.data.has_more is false }}",
106
+ "{{ 'next' not in headers['link'] }}",
107
+ ],
108
+ title="Stop Condition",
63
109
  )
64
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
110
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
65
111
 
66
112
 
67
113
  class CustomAuthenticator(BaseModel):
68
114
  class Config:
69
115
  extra = Extra.allow
70
116
 
71
- type: Literal['CustomAuthenticator']
117
+ type: Literal["CustomAuthenticator"]
72
118
  class_name: str = Field(
73
119
  ...,
74
- description='Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.',
75
- examples=['source_railz.components.ShortLivedTokenAuthenticator'],
76
- title='Class Name',
120
+ description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.",
121
+ examples=["source_railz.components.ShortLivedTokenAuthenticator"],
122
+ title="Class Name",
77
123
  )
78
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
124
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
79
125
 
80
126
 
81
127
  class CustomBackoffStrategy(BaseModel):
82
128
  class Config:
83
129
  extra = Extra.allow
84
130
 
85
- type: Literal['CustomBackoffStrategy']
131
+ type: Literal["CustomBackoffStrategy"]
86
132
  class_name: str = Field(
87
133
  ...,
88
- description='Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.',
89
- examples=['source_railz.components.MyCustomBackoffStrategy'],
90
- title='Class Name',
134
+ description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.",
135
+ examples=["source_railz.components.MyCustomBackoffStrategy"],
136
+ title="Class Name",
91
137
  )
92
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
138
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
93
139
 
94
140
 
95
141
  class CustomErrorHandler(BaseModel):
96
142
  class Config:
97
143
  extra = Extra.allow
98
144
 
99
- type: Literal['CustomErrorHandler']
145
+ type: Literal["CustomErrorHandler"]
100
146
  class_name: str = Field(
101
147
  ...,
102
- description='Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.',
103
- examples=['source_railz.components.MyCustomErrorHandler'],
104
- title='Class Name',
148
+ description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.",
149
+ examples=["source_railz.components.MyCustomErrorHandler"],
150
+ title="Class Name",
105
151
  )
106
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
152
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
107
153
 
108
154
 
109
155
  class CustomIncrementalSync(BaseModel):
110
156
  class Config:
111
157
  extra = Extra.allow
112
158
 
113
- type: Literal['CustomIncrementalSync']
159
+ type: Literal["CustomIncrementalSync"]
114
160
  class_name: str = Field(
115
161
  ...,
116
- description='Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.',
117
- examples=['source_railz.components.MyCustomIncrementalSync'],
118
- title='Class Name',
162
+ description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.",
163
+ examples=["source_railz.components.MyCustomIncrementalSync"],
164
+ title="Class Name",
119
165
  )
120
166
  cursor_field: str = Field(
121
167
  ...,
122
- description='The location of the value on a record that will be used as a bookmark during sync.',
168
+ description="The location of the value on a record that will be used as a bookmark during sync.",
123
169
  )
124
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
170
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
125
171
 
126
172
 
127
173
  class CustomPaginationStrategy(BaseModel):
128
174
  class Config:
129
175
  extra = Extra.allow
130
176
 
131
- type: Literal['CustomPaginationStrategy']
177
+ type: Literal["CustomPaginationStrategy"]
132
178
  class_name: str = Field(
133
179
  ...,
134
- description='Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.',
135
- examples=['source_railz.components.MyCustomPaginationStrategy'],
136
- title='Class Name',
180
+ description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.",
181
+ examples=["source_railz.components.MyCustomPaginationStrategy"],
182
+ title="Class Name",
137
183
  )
138
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
184
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
139
185
 
140
186
 
141
187
  class CustomRecordExtractor(BaseModel):
142
188
  class Config:
143
189
  extra = Extra.allow
144
190
 
145
- type: Literal['CustomRecordExtractor']
191
+ type: Literal["CustomRecordExtractor"]
146
192
  class_name: str = Field(
147
193
  ...,
148
- description='Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.',
149
- examples=['source_railz.components.MyCustomRecordExtractor'],
150
- title='Class Name',
194
+ description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.",
195
+ examples=["source_railz.components.MyCustomRecordExtractor"],
196
+ title="Class Name",
151
197
  )
152
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
198
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
153
199
 
154
200
 
155
201
  class CustomRecordFilter(BaseModel):
156
202
  class Config:
157
203
  extra = Extra.allow
158
204
 
159
- type: Literal['CustomRecordFilter']
205
+ type: Literal["CustomRecordFilter"]
160
206
  class_name: str = Field(
161
207
  ...,
162
- description='Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.',
163
- examples=['source_railz.components.MyCustomCustomRecordFilter'],
164
- title='Class Name',
208
+ description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.",
209
+ examples=["source_railz.components.MyCustomCustomRecordFilter"],
210
+ title="Class Name",
165
211
  )
166
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
212
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
167
213
 
168
214
 
169
215
  class CustomRequester(BaseModel):
170
216
  class Config:
171
217
  extra = Extra.allow
172
218
 
173
- type: Literal['CustomRequester']
219
+ type: Literal["CustomRequester"]
174
220
  class_name: str = Field(
175
221
  ...,
176
- description='Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.',
177
- examples=['source_railz.components.MyCustomRecordExtractor'],
178
- title='Class Name',
222
+ description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.",
223
+ examples=["source_railz.components.MyCustomRecordExtractor"],
224
+ title="Class Name",
179
225
  )
180
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
226
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
181
227
 
182
228
 
183
229
  class CustomRetriever(BaseModel):
184
230
  class Config:
185
231
  extra = Extra.allow
186
232
 
187
- type: Literal['CustomRetriever']
233
+ type: Literal["CustomRetriever"]
188
234
  class_name: str = Field(
189
235
  ...,
190
- description='Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.',
191
- examples=['source_railz.components.MyCustomRetriever'],
192
- title='Class Name',
236
+ description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.",
237
+ examples=["source_railz.components.MyCustomRetriever"],
238
+ title="Class Name",
193
239
  )
194
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
240
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
195
241
 
196
242
 
197
243
  class CustomPartitionRouter(BaseModel):
198
244
  class Config:
199
245
  extra = Extra.allow
200
246
 
201
- type: Literal['CustomPartitionRouter']
247
+ type: Literal["CustomPartitionRouter"]
202
248
  class_name: str = Field(
203
249
  ...,
204
- description='Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.',
205
- examples=['source_railz.components.MyCustomPartitionRouter'],
206
- title='Class Name',
250
+ description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.",
251
+ examples=["source_railz.components.MyCustomPartitionRouter"],
252
+ title="Class Name",
207
253
  )
208
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
254
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
209
255
 
210
256
 
211
257
  class CustomSchemaLoader(BaseModel):
212
258
  class Config:
213
259
  extra = Extra.allow
214
260
 
215
- type: Literal['CustomSchemaLoader']
261
+ type: Literal["CustomSchemaLoader"]
216
262
  class_name: str = Field(
217
263
  ...,
218
- description='Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.',
219
- examples=['source_railz.components.MyCustomSchemaLoader'],
220
- title='Class Name',
264
+ description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.",
265
+ examples=["source_railz.components.MyCustomSchemaLoader"],
266
+ title="Class Name",
221
267
  )
222
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
268
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
269
+
270
+
271
+ class CustomSchemaNormalization(BaseModel):
272
+ class Config:
273
+ extra = Extra.allow
274
+
275
+ type: Literal["CustomSchemaNormalization"]
276
+ class_name: str = Field(
277
+ ...,
278
+ description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.",
279
+ examples=[
280
+ "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer"
281
+ ],
282
+ title="Class Name",
283
+ )
284
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
285
+
286
+
287
+ class CustomStateMigration(BaseModel):
288
+ class Config:
289
+ extra = Extra.allow
290
+
291
+ type: Literal["CustomStateMigration"]
292
+ class_name: str = Field(
293
+ ...,
294
+ description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.",
295
+ examples=["source_railz.components.MyCustomStateMigration"],
296
+ title="Class Name",
297
+ )
298
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
223
299
 
224
300
 
225
301
  class CustomTransformation(BaseModel):
226
302
  class Config:
227
303
  extra = Extra.allow
228
304
 
229
- type: Literal['CustomTransformation']
305
+ type: Literal["CustomTransformation"]
230
306
  class_name: str = Field(
231
307
  ...,
232
- description='Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.',
233
- examples=['source_railz.components.MyCustomTransformation'],
234
- title='Class Name',
308
+ description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.",
309
+ examples=["source_railz.components.MyCustomTransformation"],
310
+ title="Class Name",
235
311
  )
236
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
312
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
313
+
314
+
315
+ class LegacyToPerPartitionStateMigration(BaseModel):
316
+ class Config:
317
+ extra = Extra.allow
318
+
319
+ type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None
320
+
321
+
322
+ class Algorithm(Enum):
323
+ HS256 = "HS256"
324
+ HS384 = "HS384"
325
+ HS512 = "HS512"
326
+ ES256 = "ES256"
327
+ ES256K = "ES256K"
328
+ ES384 = "ES384"
329
+ ES512 = "ES512"
330
+ RS256 = "RS256"
331
+ RS384 = "RS384"
332
+ RS512 = "RS512"
333
+ PS256 = "PS256"
334
+ PS384 = "PS384"
335
+ PS512 = "PS512"
336
+ EdDSA = "EdDSA"
337
+
338
+
339
+ class JwtHeaders(BaseModel):
340
+ class Config:
341
+ extra = Extra.forbid
342
+
343
+ kid: Optional[str] = Field(
344
+ None,
345
+ description="Private key ID for user account.",
346
+ examples=["{{ config['kid'] }}"],
347
+ title="Key Identifier",
348
+ )
349
+ typ: Optional[str] = Field(
350
+ "JWT",
351
+ description="The media type of the complete JWT.",
352
+ examples=["JWT"],
353
+ title="Type",
354
+ )
355
+ cty: Optional[str] = Field(
356
+ None,
357
+ description="Content type of JWT header.",
358
+ examples=["JWT"],
359
+ title="Content Type",
360
+ )
361
+
362
+
363
+ class JwtPayload(BaseModel):
364
+ class Config:
365
+ extra = Extra.forbid
366
+
367
+ iss: Optional[str] = Field(
368
+ None,
369
+ description="The user/principal that issued the JWT. Commonly a value unique to the user.",
370
+ examples=["{{ config['iss'] }}"],
371
+ title="Issuer",
372
+ )
373
+ sub: Optional[str] = Field(
374
+ None,
375
+ description="The subject of the JWT. Commonly defined by the API.",
376
+ title="Subject",
377
+ )
378
+ aud: Optional[str] = Field(
379
+ None,
380
+ description="The recipient that the JWT is intended for. Commonly defined by the API.",
381
+ examples=["appstoreconnect-v1"],
382
+ title="Audience",
383
+ )
384
+
385
+
386
+ class JwtAuthenticator(BaseModel):
387
+ type: Literal["JwtAuthenticator"]
388
+ secret_key: str = Field(
389
+ ...,
390
+ description="Secret used to sign the JSON web token.",
391
+ examples=["{{ config['secret_key'] }}"],
392
+ )
393
+ base64_encode_secret_key: Optional[bool] = Field(
394
+ False,
395
+ description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.',
396
+ )
397
+ algorithm: Algorithm = Field(
398
+ ...,
399
+ description="Algorithm used to sign the JSON web token.",
400
+ examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"],
401
+ )
402
+ token_duration: Optional[int] = Field(
403
+ 1200,
404
+ description="The amount of time in seconds a JWT token can be valid after being issued.",
405
+ examples=[1200, 3600],
406
+ title="Token Duration",
407
+ )
408
+ header_prefix: Optional[str] = Field(
409
+ None,
410
+ description="The prefix to be used within the Authentication header.",
411
+ examples=["Bearer", "Basic"],
412
+ title="Header Prefix",
413
+ )
414
+ jwt_headers: Optional[JwtHeaders] = Field(
415
+ None,
416
+ description="JWT headers used when signing JSON web token.",
417
+ title="JWT Headers",
418
+ )
419
+ additional_jwt_headers: Optional[Dict[str, Any]] = Field(
420
+ None,
421
+ description="Additional headers to be included with the JWT headers object.",
422
+ title="Additional JWT Headers",
423
+ )
424
+ jwt_payload: Optional[JwtPayload] = Field(
425
+ None,
426
+ description="JWT Payload used when signing JSON web token.",
427
+ title="JWT Payload",
428
+ )
429
+ additional_jwt_payload: Optional[Dict[str, Any]] = Field(
430
+ None,
431
+ description="Additional properties to be added to the JWT payload.",
432
+ title="Additional JWT Payload Properties",
433
+ )
434
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
237
435
 
238
436
 
239
437
  class RefreshTokenUpdater(BaseModel):
240
438
  refresh_token_name: Optional[str] = Field(
241
- 'refresh_token',
242
- description='The name of the property which contains the updated refresh token in the response from the token refresh endpoint.',
243
- examples=['refresh_token'],
244
- title='Refresh Token Property Name',
439
+ "refresh_token",
440
+ description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.",
441
+ examples=["refresh_token"],
442
+ title="Refresh Token Property Name",
245
443
  )
246
444
  access_token_config_path: Optional[List[str]] = Field(
247
- ['credentials', 'access_token'],
248
- description='Config path to the access token. Make sure the field actually exists in the config.',
249
- examples=[['credentials', 'access_token'], ['access_token']],
250
- title='Config Path To Access Token',
445
+ ["credentials", "access_token"],
446
+ description="Config path to the access token. Make sure the field actually exists in the config.",
447
+ examples=[["credentials", "access_token"], ["access_token"]],
448
+ title="Config Path To Access Token",
251
449
  )
252
450
  refresh_token_config_path: Optional[List[str]] = Field(
253
- ['credentials', 'refresh_token'],
254
- description='Config path to the access token. Make sure the field actually exists in the config.',
255
- examples=[['credentials', 'refresh_token'], ['refresh_token']],
256
- title='Config Path To Refresh Token',
451
+ ["credentials", "refresh_token"],
452
+ description="Config path to the access token. Make sure the field actually exists in the config.",
453
+ examples=[["credentials", "refresh_token"], ["refresh_token"]],
454
+ title="Config Path To Refresh Token",
257
455
  )
258
456
  token_expiry_date_config_path: Optional[List[str]] = Field(
259
- ['credentials', 'token_expiry_date'],
260
- description='Config path to the expiry date. Make sure actually exists in the config.',
261
- examples=[['credentials', 'token_expiry_date']],
262
- title='Config Path To Expiry Date',
457
+ ["credentials", "token_expiry_date"],
458
+ description="Config path to the expiry date. Make sure actually exists in the config.",
459
+ examples=[["credentials", "token_expiry_date"]],
460
+ title="Config Path To Expiry Date",
461
+ )
462
+ refresh_token_error_status_codes: Optional[List[int]] = Field(
463
+ [],
464
+ description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error",
465
+ examples=[[400, 500]],
466
+ title="Refresh Token Error Status Codes",
467
+ )
468
+ refresh_token_error_key: Optional[str] = Field(
469
+ "",
470
+ description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).",
471
+ examples=["error"],
472
+ title="Refresh Token Error Key",
473
+ )
474
+ refresh_token_error_values: Optional[List[str]] = Field(
475
+ [],
476
+ description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error',
477
+ examples=[["invalid_grant", "invalid_permissions"]],
478
+ title="Refresh Token Error Values",
263
479
  )
264
480
 
265
481
 
266
482
  class OAuthAuthenticator(BaseModel):
267
- type: Literal['OAuthAuthenticator']
483
+ type: Literal["OAuthAuthenticator"]
268
484
  client_id: str = Field(
269
485
  ...,
270
- description='The OAuth client ID. Fill it in the user inputs.',
486
+ description="The OAuth client ID. Fill it in the user inputs.",
271
487
  examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"],
272
- title='Client ID',
488
+ title="Client ID",
273
489
  )
274
490
  client_secret: str = Field(
275
491
  ...,
276
- description='The OAuth client secret. Fill it in the user inputs.',
492
+ description="The OAuth client secret. Fill it in the user inputs.",
277
493
  examples=[
278
494
  "{{ config['client_secret }}",
279
495
  "{{ config['credentials']['client_secret }}",
280
496
  ],
281
- title='Client Secret',
497
+ title="Client Secret",
282
498
  )
283
499
  refresh_token: Optional[str] = Field(
284
500
  None,
285
- description='Credential artifact used to get a new access token.',
501
+ description="Credential artifact used to get a new access token.",
286
502
  examples=[
287
503
  "{{ config['refresh_token'] }}",
288
504
  "{{ config['credentials]['refresh_token'] }}",
289
505
  ],
290
- title='Refresh Token',
506
+ title="Refresh Token",
291
507
  )
292
- token_refresh_endpoint: str = Field(
293
- ...,
294
- description='The full URL to call to obtain a new access token.',
295
- examples=['https://connect.squareup.com/oauth2/token'],
296
- title='Token Refresh Endpoint',
508
+ token_refresh_endpoint: Optional[str] = Field(
509
+ None,
510
+ description="The full URL to call to obtain a new access token.",
511
+ examples=["https://connect.squareup.com/oauth2/token"],
512
+ title="Token Refresh Endpoint",
297
513
  )
298
514
  access_token_name: Optional[str] = Field(
299
- 'access_token',
300
- description='The name of the property which contains the access token in the response from the token refresh endpoint.',
301
- examples=['access_token'],
302
- title='Access Token Property Name',
515
+ "access_token",
516
+ description="The name of the property which contains the access token in the response from the token refresh endpoint.",
517
+ examples=["access_token"],
518
+ title="Access Token Property Name",
519
+ )
520
+ access_token_value: Optional[str] = Field(
521
+ None,
522
+ description="The value of the access_token to bypass the token refreshing using `refresh_token`.",
523
+ examples=["secret_access_token_value"],
524
+ title="Access Token Value",
303
525
  )
304
526
  expires_in_name: Optional[str] = Field(
305
- 'expires_in',
306
- description='The name of the property which contains the expiry date in the response from the token refresh endpoint.',
307
- examples=['expires_in'],
308
- title='Token Expiry Property Name',
527
+ "expires_in",
528
+ description="The name of the property which contains the expiry date in the response from the token refresh endpoint.",
529
+ examples=["expires_in"],
530
+ title="Token Expiry Property Name",
309
531
  )
310
532
  grant_type: Optional[str] = Field(
311
- 'refresh_token',
312
- description='Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.',
313
- examples=['refresh_token', 'client_credentials'],
314
- title='Grant Type',
533
+ "refresh_token",
534
+ description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
535
+ examples=["refresh_token", "client_credentials"],
536
+ title="Grant Type",
315
537
  )
316
538
  refresh_request_body: Optional[Dict[str, Any]] = Field(
317
539
  None,
318
- description='Body of the request sent to get a new access token.',
540
+ description="Body of the request sent to get a new access token.",
319
541
  examples=[
320
542
  {
321
- 'applicationId': "{{ config['application_id'] }}",
322
- 'applicationSecret': "{{ config['application_secret'] }}",
323
- 'token': "{{ config['token'] }}",
543
+ "applicationId": "{{ config['application_id'] }}",
544
+ "applicationSecret": "{{ config['application_secret'] }}",
545
+ "token": "{{ config['token'] }}",
324
546
  }
325
547
  ],
326
- title='Refresh Request Body',
548
+ title="Refresh Request Body",
327
549
  )
328
550
  scopes: Optional[List[str]] = Field(
329
551
  None,
330
- description='List of scopes that should be granted to the access token.',
331
- examples=[
332
- ['crm.list.read', 'crm.objects.contacts.read', 'crm.schema.contacts.read']
333
- ],
334
- title='Scopes',
552
+ description="List of scopes that should be granted to the access token.",
553
+ examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]],
554
+ title="Scopes",
335
555
  )
336
556
  token_expiry_date: Optional[str] = Field(
337
557
  None,
338
- description='The access token expiry date.',
339
- examples=['2023-04-06T07:12:10.421833+00:00', 1680842386],
340
- title='Token Expiry Date',
558
+ description="The access token expiry date.",
559
+ examples=["2023-04-06T07:12:10.421833+00:00", 1680842386],
560
+ title="Token Expiry Date",
341
561
  )
342
562
  token_expiry_date_format: Optional[str] = Field(
343
563
  None,
344
- description='The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.',
345
- examples=['%Y-%m-%d %H:%M:%S.%f+00:00'],
346
- title='Token Expiry Date Format',
564
+ description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.",
565
+ examples=["%Y-%m-%d %H:%M:%S.%f+00:00"],
566
+ title="Token Expiry Date Format",
347
567
  )
348
568
  refresh_token_updater: Optional[RefreshTokenUpdater] = Field(
349
569
  None,
350
- description='When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.',
351
- title='Token Updater',
570
+ description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.",
571
+ title="Token Updater",
352
572
  )
353
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
573
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
574
+
575
+
576
+ class DpathExtractor(BaseModel):
577
+ type: Literal["DpathExtractor"]
578
+ field_path: List[str] = Field(
579
+ ...,
580
+ description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
581
+ examples=[
582
+ ["data"],
583
+ ["data", "records"],
584
+ ["data", "{{ parameters.name }}"],
585
+ ["data", "*", "record"],
586
+ ],
587
+ title="Field Path",
588
+ )
589
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
590
+
591
+
592
+ class ResponseToFileExtractor(BaseModel):
593
+ type: Literal["ResponseToFileExtractor"]
594
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
354
595
 
355
596
 
356
597
  class ExponentialBackoffStrategy(BaseModel):
357
- type: Literal['ExponentialBackoffStrategy']
598
+ type: Literal["ExponentialBackoffStrategy"]
358
599
  factor: Optional[Union[float, str]] = Field(
359
600
  5,
360
- description='Multiplicative constant applied on each retry.',
361
- examples=[5, 5.5, '10'],
362
- title='Factor',
601
+ description="Multiplicative constant applied on each retry.",
602
+ examples=[5, 5.5, "10"],
603
+ title="Factor",
363
604
  )
364
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
605
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
365
606
 
366
607
 
367
608
  class SessionTokenRequestBearerAuthenticator(BaseModel):
368
- type: Literal['Bearer']
609
+ type: Literal["Bearer"]
369
610
 
370
611
 
371
612
  class HttpMethod(Enum):
372
- GET = 'GET'
373
- POST = 'POST'
613
+ GET = "GET"
614
+ POST = "POST"
374
615
 
375
616
 
376
617
  class Action(Enum):
377
- SUCCESS = 'SUCCESS'
378
- FAIL = 'FAIL'
379
- RETRY = 'RETRY'
380
- IGNORE = 'IGNORE'
618
+ SUCCESS = "SUCCESS"
619
+ FAIL = "FAIL"
620
+ RETRY = "RETRY"
621
+ IGNORE = "IGNORE"
622
+ RATE_LIMITED = "RATE_LIMITED"
623
+
624
+
625
+ class FailureType(Enum):
626
+ system_error = "system_error"
627
+ config_error = "config_error"
628
+ transient_error = "transient_error"
381
629
 
382
630
 
383
631
  class HttpResponseFilter(BaseModel):
384
- type: Literal['HttpResponseFilter']
385
- action: Action = Field(
386
- ...,
387
- description='Action to execute if a response matches the filter.',
388
- examples=['SUCCESS', 'FAIL', 'RETRY', 'IGNORE'],
389
- title='Action',
632
+ type: Literal["HttpResponseFilter"]
633
+ action: Optional[Action] = Field(
634
+ None,
635
+ description="Action to execute if a response matches the filter.",
636
+ examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"],
637
+ title="Action",
638
+ )
639
+ failure_type: Optional[FailureType] = Field(
640
+ None,
641
+ description="Failure type of traced exception if a response matches the filter.",
642
+ examples=["system_error", "config_error", "transient_error"],
643
+ title="Failure Type",
390
644
  )
391
645
  error_message: Optional[str] = Field(
392
646
  None,
393
- description='Error Message to display if the response matches the filter.',
394
- title='Error Message',
647
+ description="Error Message to display if the response matches the filter.",
648
+ title="Error Message",
395
649
  )
396
650
  error_message_contains: Optional[str] = Field(
397
651
  None,
398
- description='Match the response if its error message contains the substring.',
399
- example=['This API operation is not enabled for this site'],
400
- title='Error Message Substring',
652
+ description="Match the response if its error message contains the substring.",
653
+ example=["This API operation is not enabled for this site"],
654
+ title="Error Message Substring",
401
655
  )
402
656
  http_codes: Optional[List[int]] = Field(
403
657
  None,
404
- description='Match the response if its HTTP code is included in this list.',
658
+ description="Match the response if its HTTP code is included in this list.",
405
659
  examples=[[420, 429], [500]],
406
- title='HTTP Codes',
660
+ title="HTTP Codes",
661
+ unique_items=True,
407
662
  )
408
663
  predicate: Optional[str] = Field(
409
664
  None,
410
- description='Match the response if the predicate evaluates to true.',
665
+ description="Match the response if the predicate evaluates to true.",
411
666
  examples=[
412
667
  "{{ 'Too much requests' in response }}",
413
668
  "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}",
414
669
  ],
415
- title='Predicate',
670
+ title="Predicate",
416
671
  )
417
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
672
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
673
+
674
+
675
+ class TypesMap(BaseModel):
676
+ target_type: Union[str, List[str]]
677
+ current_type: Union[str, List[str]]
678
+
679
+
680
+ class SchemaTypeIdentifier(BaseModel):
681
+ type: Optional[Literal["SchemaTypeIdentifier"]] = None
682
+ schema_pointer: Optional[List[str]] = Field(
683
+ [],
684
+ description="List of nested fields defining the schema field path to extract. Defaults to [].",
685
+ title="Schema Path",
686
+ )
687
+ key_pointer: List[str] = Field(
688
+ ...,
689
+ description="List of potentially nested fields describing the full path of the field key to extract.",
690
+ title="Key Path",
691
+ )
692
+ type_pointer: Optional[List[str]] = Field(
693
+ None,
694
+ description="List of potentially nested fields describing the full path of the field type to extract.",
695
+ title="Type Path",
696
+ )
697
+ types_mapping: Optional[List[TypesMap]] = None
698
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
418
699
 
419
700
 
420
701
  class InlineSchemaLoader(BaseModel):
421
- type: Literal['InlineSchemaLoader']
702
+ type: Literal["InlineSchemaLoader"]
422
703
  schema_: Optional[Dict[str, Any]] = Field(
423
704
  None,
424
- alias='schema',
705
+ alias="schema",
425
706
  description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.',
426
- title='Schema',
707
+ title="Schema",
427
708
  )
428
709
 
429
710
 
430
711
  class JsonFileSchemaLoader(BaseModel):
431
- type: Literal['JsonFileSchemaLoader']
712
+ type: Literal["JsonFileSchemaLoader"]
432
713
  file_path: Optional[str] = Field(
433
714
  None,
434
715
  description="Path to the JSON file defining the schema. The path is relative to the connector module's root.",
435
- example=['./schemas/users.json'],
436
- title='File Path',
716
+ example=["./schemas/users.json"],
717
+ title="File Path",
437
718
  )
438
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
719
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
439
720
 
440
721
 
441
722
  class JsonDecoder(BaseModel):
442
- type: Literal['JsonDecoder']
723
+ type: Literal["JsonDecoder"]
724
+
725
+
726
+ class JsonlDecoder(BaseModel):
727
+ type: Literal["JsonlDecoder"]
728
+
729
+
730
+ class KeysToLower(BaseModel):
731
+ type: Literal["KeysToLower"]
732
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
733
+
734
+
735
+ class KeysToSnakeCase(BaseModel):
736
+ type: Literal["KeysToSnakeCase"]
737
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
738
+
739
+
740
+ class KeysReplace(BaseModel):
741
+ type: Literal["KeysReplace"]
742
+ old: str = Field(
743
+ ...,
744
+ description="Old value to replace.",
745
+ examples=[" ", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
746
+ title="Old value",
747
+ )
748
+ new: str = Field(
749
+ ...,
750
+ description="New value to set.",
751
+ examples=["_", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
752
+ title="New value",
753
+ )
754
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
755
+
756
+
757
+ class FlattenFields(BaseModel):
758
+ type: Literal["FlattenFields"]
759
+ flatten_lists: Optional[bool] = Field(
760
+ True,
761
+ description="Whether to flatten lists or leave it as is. Default is True.",
762
+ title="Flatten Lists",
763
+ )
764
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
765
+
766
+
767
+ class IterableDecoder(BaseModel):
768
+ type: Literal["IterableDecoder"]
769
+
770
+
771
+ class XmlDecoder(BaseModel):
772
+ type: Literal["XmlDecoder"]
773
+
774
+
775
+ class CustomDecoder(BaseModel):
776
+ class Config:
777
+ extra = Extra.allow
778
+
779
+ type: Literal["CustomDecoder"]
780
+ class_name: str = Field(
781
+ ...,
782
+ description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.",
783
+ examples=["source_amazon_ads.components.GzipJsonlDecoder"],
784
+ title="Class Name",
785
+ )
786
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
787
+
788
+
789
+ class GzipJsonDecoder(BaseModel):
790
+ class Config:
791
+ extra = Extra.allow
792
+
793
+ type: Literal["GzipJsonDecoder"]
794
+ encoding: Optional[str] = "utf-8"
795
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
443
796
 
444
797
 
445
798
  class MinMaxDatetime(BaseModel):
446
- type: Literal['MinMaxDatetime']
799
+ type: Literal["MinMaxDatetime"]
447
800
  datetime: str = Field(
448
801
  ...,
449
- description='Datetime value.',
450
- examples=['2021-01-01', '2021-01-01T00:00:00Z', "{{ config['start_time'] }}"],
451
- title='Datetime',
802
+ description="Datetime value.",
803
+ examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"],
804
+ title="Datetime",
452
805
  )
453
806
  datetime_format: Optional[str] = Field(
454
- '',
455
- description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
456
- examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s'],
457
- title='Datetime Format',
807
+ "",
808
+ description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
809
+ examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
810
+ title="Datetime Format",
458
811
  )
459
812
  max_datetime: Optional[str] = Field(
460
813
  None,
461
- description='Ceiling applied on the datetime value. Must be formatted with the datetime_format field.',
462
- examples=['2021-01-01T00:00:00Z', '2021-01-01'],
463
- title='Max Datetime',
814
+ description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.",
815
+ examples=["2021-01-01T00:00:00Z", "2021-01-01"],
816
+ title="Max Datetime",
464
817
  )
465
818
  min_datetime: Optional[str] = Field(
466
819
  None,
467
- description='Floor applied on the datetime value. Must be formatted with the datetime_format field.',
468
- examples=['2010-01-01T00:00:00Z', '2010-01-01'],
469
- title='Min Datetime',
820
+ description="Floor applied on the datetime value. Must be formatted with the datetime_format field.",
821
+ examples=["2010-01-01T00:00:00Z", "2010-01-01"],
822
+ title="Min Datetime",
470
823
  )
471
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
824
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
472
825
 
473
826
 
474
827
  class NoAuth(BaseModel):
475
- type: Literal['NoAuth']
476
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
828
+ type: Literal["NoAuth"]
829
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
477
830
 
478
831
 
479
832
  class NoPagination(BaseModel):
480
- type: Literal['NoPagination']
833
+ type: Literal["NoPagination"]
834
+
835
+
836
+ class State(BaseModel):
837
+ class Config:
838
+ extra = Extra.allow
839
+
840
+ min: int
841
+ max: int
842
+
843
+
844
+ class OauthConnectorInputSpecification(BaseModel):
845
+ class Config:
846
+ extra = Extra.allow
847
+
848
+ consent_url: str = Field(
849
+ ...,
850
+ description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
851
+ examples=[
852
+ "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
853
+ "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}",
854
+ ],
855
+ title="Consent URL",
856
+ )
857
+ scope: Optional[str] = Field(
858
+ None,
859
+ description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
860
+ examples=["user:read user:read_orders workspaces:read"],
861
+ title="Scopes",
862
+ )
863
+ access_token_url: str = Field(
864
+ ...,
865
+ description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
866
+ examples=[
867
+ "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
868
+ ],
869
+ title="Access Token URL",
870
+ )
871
+ access_token_headers: Optional[Dict[str, Any]] = Field(
872
+ None,
873
+ description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
874
+ examples=[{"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"}],
875
+ title="Access Token Headers",
876
+ )
877
+ access_token_params: Optional[Dict[str, Any]] = Field(
878
+ None,
879
+ description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
880
+ examples=[
881
+ {
882
+ "{auth_code_key}": "{{auth_code_key}}",
883
+ "{client_id_key}": "{{client_id_key}}",
884
+ "{client_secret_key}": "{{client_secret_key}}",
885
+ }
886
+ ],
887
+ title="Access Token Query Params (Json Encoded)",
888
+ )
889
+ extract_output: List[str] = Field(
890
+ ...,
891
+ description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.",
892
+ examples=[["access_token", "refresh_token", "other_field"]],
893
+ title="Extract Output",
894
+ )
895
+ state: Optional[State] = Field(
896
+ None,
897
+ description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.",
898
+ examples=[{"min": 7, "max": 128}],
899
+ title="Configurable State Query Param",
900
+ )
901
+ client_id_key: Optional[str] = Field(
902
+ None,
903
+ description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
904
+ examples=["my_custom_client_id_key_name"],
905
+ title="Client ID Key Override",
906
+ )
907
+ client_secret_key: Optional[str] = Field(
908
+ None,
909
+ description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
910
+ examples=["my_custom_client_secret_key_name"],
911
+ title="Client Secret Key Override",
912
+ )
913
+ scope_key: Optional[str] = Field(
914
+ None,
915
+ description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
916
+ examples=["my_custom_scope_key_key_name"],
917
+ title="Scopes Key Override",
918
+ )
919
+ state_key: Optional[str] = Field(
920
+ None,
921
+ description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.",
922
+ examples=["my_custom_state_key_key_name"],
923
+ title="State Key Override",
924
+ )
925
+ auth_code_key: Optional[str] = Field(
926
+ None,
927
+ description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.",
928
+ examples=["my_custom_auth_code_key_name"],
929
+ title="Auth Code Key Override",
930
+ )
931
+ redirect_uri_key: Optional[str] = Field(
932
+ None,
933
+ description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
934
+ examples=["my_custom_redirect_uri_key_name"],
935
+ title="Redirect URI Key Override",
936
+ )
481
937
 
482
938
 
483
939
  class OAuthConfigSpecification(BaseModel):
484
940
  class Config:
485
941
  extra = Extra.allow
486
942
 
487
- oauth_user_input_from_connector_config_specification: Optional[
488
- Dict[str, Any]
489
- ] = Field(
943
+ oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field(
490
944
  None,
491
945
  description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }",
492
946
  examples=[
493
- {'app_id': {'type': 'string', 'path_in_connector_config': ['app_id']}},
947
+ {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}},
494
948
  {
495
- 'app_id': {
496
- 'type': 'string',
497
- 'path_in_connector_config': ['info', 'app_id'],
949
+ "app_id": {
950
+ "type": "string",
951
+ "path_in_connector_config": ["info", "app_id"],
498
952
  }
499
953
  },
500
954
  ],
501
- title='OAuth user input',
955
+ title="OAuth user input",
956
+ )
957
+ oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
958
+ None,
959
+ description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{auth_code_key}": "{{auth_code_key}}",\n "{client_id_key}": "{{client_id_key}}",\n "{client_secret_key}": "{{client_secret_key}}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }',
960
+ title="DeclarativeOAuth Connector Specification",
502
961
  )
503
962
  complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
504
963
  None,
505
964
  description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
506
965
  examples=[
507
966
  {
508
- 'refresh_token': {
509
- 'type': 'string,',
510
- 'path_in_connector_config': ['credentials', 'refresh_token'],
967
+ "refresh_token": {
968
+ "type": "string,",
969
+ "path_in_connector_config": ["credentials", "refresh_token"],
511
970
  }
512
971
  }
513
972
  ],
514
- title='OAuth output specification',
973
+ title="OAuth output specification",
515
974
  )
516
975
  complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field(
517
976
  None,
518
- description='OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }',
519
- examples=[
520
- {'client_id': {'type': 'string'}, 'client_secret': {'type': 'string'}}
521
- ],
522
- title='OAuth input specification',
977
+ description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }",
978
+ examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}],
979
+ title="OAuth input specification",
523
980
  )
524
981
  complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field(
525
982
  None,
526
983
  description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }",
527
984
  examples=[
528
985
  {
529
- 'client_id': {
530
- 'type': 'string,',
531
- 'path_in_connector_config': ['credentials', 'client_id'],
986
+ "client_id": {
987
+ "type": "string,",
988
+ "path_in_connector_config": ["credentials", "client_id"],
532
989
  },
533
- 'client_secret': {
534
- 'type': 'string,',
535
- 'path_in_connector_config': ['credentials', 'client_secret'],
990
+ "client_secret": {
991
+ "type": "string,",
992
+ "path_in_connector_config": ["credentials", "client_secret"],
536
993
  },
537
994
  }
538
995
  ],
539
- title='OAuth server output specification',
996
+ title="OAuth server output specification",
540
997
  )
541
998
 
542
999
 
543
1000
  class OffsetIncrement(BaseModel):
544
- type: Literal['OffsetIncrement']
1001
+ type: Literal["OffsetIncrement"]
545
1002
  page_size: Optional[Union[int, str]] = Field(
546
1003
  None,
547
- description='The number of records to include in each pages.',
1004
+ description="The number of records to include in each pages.",
548
1005
  examples=[100, "{{ config['page_size'] }}"],
549
- title='Limit',
1006
+ title="Limit",
550
1007
  )
551
1008
  inject_on_first_request: Optional[bool] = Field(
552
1009
  False,
553
- description='Using the `offset` with value `0` during the first request',
554
- title='Inject Offset',
1010
+ description="Using the `offset` with value `0` during the first request",
1011
+ title="Inject Offset",
555
1012
  )
556
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1013
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
557
1014
 
558
1015
 
559
1016
  class PageIncrement(BaseModel):
560
- type: Literal['PageIncrement']
1017
+ type: Literal["PageIncrement"]
561
1018
  page_size: Optional[Union[int, str]] = Field(
562
1019
  None,
563
- description='The number of records to include in each pages.',
564
- examples=[100, '100', "{{ config['page_size'] }}"],
565
- title='Page Size',
1020
+ description="The number of records to include in each pages.",
1021
+ examples=[100, "100", "{{ config['page_size'] }}"],
1022
+ title="Page Size",
566
1023
  )
567
1024
  start_from_page: Optional[int] = Field(
568
1025
  0,
569
- description='Index of the first page to request.',
1026
+ description="Index of the first page to request.",
570
1027
  examples=[0, 1],
571
- title='Start From Page',
1028
+ title="Start From Page",
572
1029
  )
573
1030
  inject_on_first_request: Optional[bool] = Field(
574
1031
  False,
575
- description='Using the `page number` with value defined by `start_from_page` during the first request',
576
- title='Inject Page Number',
1032
+ description="Using the `page number` with value defined by `start_from_page` during the first request",
1033
+ title="Inject Page Number",
577
1034
  )
578
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1035
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
579
1036
 
580
1037
 
581
1038
  class PrimaryKey(BaseModel):
582
1039
  __root__: Union[str, List[str], List[List[str]]] = Field(
583
1040
  ...,
584
- description='The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.',
585
- examples=['id', ['code', 'type']],
586
- title='Primary Key',
1041
+ description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.",
1042
+ examples=["id", ["code", "type"]],
1043
+ title="Primary Key",
587
1044
  )
588
1045
 
589
1046
 
590
1047
  class RecordFilter(BaseModel):
591
- type: Literal['RecordFilter']
1048
+ type: Literal["RecordFilter"]
592
1049
  condition: Optional[str] = Field(
593
- '',
594
- description='The predicate to filter a record. Records will be removed if evaluated to False.',
1050
+ "",
1051
+ description="The predicate to filter a record. Records will be removed if evaluated to False.",
595
1052
  examples=[
596
1053
  "{{ record['created_at'] >= stream_interval['start_time'] }}",
597
1054
  "{{ record.status in ['active', 'expired'] }}",
598
1055
  ],
599
1056
  )
600
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1057
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
601
1058
 
602
1059
 
603
1060
  class SchemaNormalization(Enum):
604
- None_ = 'None'
605
- Default = 'Default'
1061
+ None_ = "None"
1062
+ Default = "Default"
606
1063
 
607
1064
 
608
1065
  class RemoveFields(BaseModel):
609
- type: Literal['RemoveFields']
1066
+ type: Literal["RemoveFields"]
610
1067
  condition: Optional[str] = Field(
611
- '',
612
- description='The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,',
1068
+ "",
1069
+ description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,",
613
1070
  examples=[
614
1071
  "{{ property|string == '' }}",
615
- '{{ property is integer }}',
616
- '{{ property|length > 5 }}',
1072
+ "{{ property is integer }}",
1073
+ "{{ property|length > 5 }}",
617
1074
  "{{ property == 'some_string_to_match' }}",
618
1075
  ],
619
1076
  )
620
1077
  field_pointers: List[List[str]] = Field(
621
1078
  ...,
622
- description='Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.',
623
- examples=[['tags'], [['content', 'html'], ['content', 'plain_text']]],
624
- title='Field Paths',
1079
+ description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.",
1080
+ examples=[["tags"], [["content", "html"], ["content", "plain_text"]]],
1081
+ title="Field Paths",
625
1082
  )
626
1083
 
627
1084
 
628
1085
  class RequestPath(BaseModel):
629
- type: Literal['RequestPath']
1086
+ type: Literal["RequestPath"]
630
1087
 
631
1088
 
632
1089
  class InjectInto(Enum):
633
- request_parameter = 'request_parameter'
634
- header = 'header'
635
- body_data = 'body_data'
636
- body_json = 'body_json'
1090
+ request_parameter = "request_parameter"
1091
+ header = "header"
1092
+ body_data = "body_data"
1093
+ body_json = "body_json"
637
1094
 
638
1095
 
639
1096
  class RequestOption(BaseModel):
640
- type: Literal['RequestOption']
1097
+ type: Literal["RequestOption"]
641
1098
  field_name: str = Field(
642
1099
  ...,
643
- description='Configures which key should be used in the location that the descriptor is being injected into',
644
- examples=['segment_id'],
645
- title='Request Option',
1100
+ description="Configures which key should be used in the location that the descriptor is being injected into",
1101
+ examples=["segment_id"],
1102
+ title="Request Option",
646
1103
  )
647
1104
  inject_into: InjectInto = Field(
648
1105
  ...,
649
- description='Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.',
650
- examples=['request_parameter', 'header', 'body_data', 'body_json'],
651
- title='Inject Into',
1106
+ description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.",
1107
+ examples=["request_parameter", "header", "body_data", "body_json"],
1108
+ title="Inject Into",
652
1109
  )
653
1110
 
654
1111
 
@@ -660,106 +1117,184 @@ class Schemas(BaseModel):
660
1117
 
661
1118
 
662
1119
  class LegacySessionTokenAuthenticator(BaseModel):
663
- type: Literal['LegacySessionTokenAuthenticator']
1120
+ type: Literal["LegacySessionTokenAuthenticator"]
664
1121
  header: str = Field(
665
1122
  ...,
666
- description='The name of the session token header that will be injected in the request',
667
- examples=['X-Session'],
668
- title='Session Request Header',
1123
+ description="The name of the session token header that will be injected in the request",
1124
+ examples=["X-Session"],
1125
+ title="Session Request Header",
669
1126
  )
670
1127
  login_url: str = Field(
671
1128
  ...,
672
- description='Path of the login URL (do not include the base URL)',
673
- examples=['session'],
674
- title='Login Path',
1129
+ description="Path of the login URL (do not include the base URL)",
1130
+ examples=["session"],
1131
+ title="Login Path",
675
1132
  )
676
1133
  session_token: Optional[str] = Field(
677
1134
  None,
678
- description='Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair',
1135
+ description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair",
679
1136
  example=["{{ config['session_token'] }}"],
680
- title='Session Token',
1137
+ title="Session Token",
681
1138
  )
682
1139
  session_token_response_key: str = Field(
683
1140
  ...,
684
- description='Name of the key of the session token to be extracted from the response',
685
- examples=['id'],
686
- title='Response Token Response Key',
1141
+ description="Name of the key of the session token to be extracted from the response",
1142
+ examples=["id"],
1143
+ title="Response Token Response Key",
687
1144
  )
688
1145
  username: Optional[str] = Field(
689
1146
  None,
690
- description='Username used to authenticate and obtain a session token',
1147
+ description="Username used to authenticate and obtain a session token",
691
1148
  examples=[" {{ config['username'] }}"],
692
- title='Username',
1149
+ title="Username",
693
1150
  )
694
1151
  password: Optional[str] = Field(
695
- '',
696
- description='Password used to authenticate and obtain a session token',
697
- examples=["{{ config['password'] }}", ''],
698
- title='Password',
1152
+ "",
1153
+ description="Password used to authenticate and obtain a session token",
1154
+ examples=["{{ config['password'] }}", ""],
1155
+ title="Password",
699
1156
  )
700
1157
  validate_session_url: str = Field(
701
1158
  ...,
702
- description='Path of the URL to use to validate that the session token is valid (do not include the base URL)',
703
- examples=['user/current'],
704
- title='Validate Session Path',
1159
+ description="Path of the URL to use to validate that the session token is valid (do not include the base URL)",
1160
+ examples=["user/current"],
1161
+ title="Validate Session Path",
705
1162
  )
706
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1163
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1164
+
1165
+
1166
+ class JsonLineParser(BaseModel):
1167
+ type: Literal["JsonLineParser"]
1168
+ encoding: Optional[str] = "utf-8"
1169
+
1170
+
1171
+ class CsvParser(BaseModel):
1172
+ type: Literal["CsvParser"]
1173
+ encoding: Optional[str] = "utf-8"
1174
+ delimiter: Optional[str] = ","
1175
+
1176
+
1177
+ class AsyncJobStatusMap(BaseModel):
1178
+ type: Optional[Literal["AsyncJobStatusMap"]] = None
1179
+ running: List[str]
1180
+ completed: List[str]
1181
+ failed: List[str]
1182
+ timeout: List[str]
707
1183
 
708
1184
 
709
1185
  class ValueType(Enum):
710
- string = 'string'
711
- number = 'number'
712
- integer = 'integer'
713
- boolean = 'boolean'
1186
+ string = "string"
1187
+ number = "number"
1188
+ integer = "integer"
1189
+ boolean = "boolean"
714
1190
 
715
1191
 
716
1192
  class WaitTimeFromHeader(BaseModel):
717
- type: Literal['WaitTimeFromHeader']
1193
+ type: Literal["WaitTimeFromHeader"]
718
1194
  header: str = Field(
719
1195
  ...,
720
- description='The name of the response header defining how long to wait before retrying.',
721
- examples=['Retry-After'],
722
- title='Response Header Name',
1196
+ description="The name of the response header defining how long to wait before retrying.",
1197
+ examples=["Retry-After"],
1198
+ title="Response Header Name",
723
1199
  )
724
1200
  regex: Optional[str] = Field(
725
1201
  None,
726
- description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.',
727
- examples=['([-+]?\\d+)'],
728
- title='Extraction Regex',
1202
+ description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
1203
+ examples=["([-+]?\\d+)"],
1204
+ title="Extraction Regex",
1205
+ )
1206
+ max_waiting_time_in_seconds: Optional[float] = Field(
1207
+ None,
1208
+ description="Given the value extracted from the header is greater than this value, stop the stream.",
1209
+ examples=[3600],
1210
+ title="Max Waiting Time in Seconds",
729
1211
  )
730
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1212
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
731
1213
 
732
1214
 
733
1215
  class WaitUntilTimeFromHeader(BaseModel):
734
- type: Literal['WaitUntilTimeFromHeader']
1216
+ type: Literal["WaitUntilTimeFromHeader"]
735
1217
  header: str = Field(
736
1218
  ...,
737
- description='The name of the response header defining how long to wait before retrying.',
738
- examples=['wait_time'],
739
- title='Response Header',
1219
+ description="The name of the response header defining how long to wait before retrying.",
1220
+ examples=["wait_time"],
1221
+ title="Response Header",
740
1222
  )
741
1223
  min_wait: Optional[Union[float, str]] = Field(
742
1224
  None,
743
- description='Minimum time to wait before retrying.',
744
- examples=[10, '60'],
745
- title='Minimum Wait Time',
1225
+ description="Minimum time to wait before retrying.",
1226
+ examples=[10, "60"],
1227
+ title="Minimum Wait Time",
746
1228
  )
747
1229
  regex: Optional[str] = Field(
748
1230
  None,
749
- description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.',
750
- examples=['([-+]?\\d+)'],
751
- title='Extraction Regex',
1231
+ description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
1232
+ examples=["([-+]?\\d+)"],
1233
+ title="Extraction Regex",
752
1234
  )
753
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1235
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1236
+
1237
+
1238
+ class ComponentMappingDefinition(BaseModel):
1239
+ type: Literal["ComponentMappingDefinition"]
1240
+ field_path: List[str] = Field(
1241
+ ...,
1242
+ description="A list of potentially nested fields indicating the full path where value will be added or updated.",
1243
+ examples=[
1244
+ ["data"],
1245
+ ["data", "records"],
1246
+ ["data", 1, "name"],
1247
+ ["data", "{{ components_values.name }}"],
1248
+ ["data", "*", "record"],
1249
+ ["*", "**", "name"],
1250
+ ],
1251
+ title="Field Path",
1252
+ )
1253
+ value: str = Field(
1254
+ ...,
1255
+ description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.",
1256
+ examples=[
1257
+ "{{ components_values['updates'] }}",
1258
+ "{{ components_values['MetaData']['LastUpdatedTime'] }}",
1259
+ "{{ config['segment_id'] }}",
1260
+ "{{ stream_slice['parent_id'] }}",
1261
+ "{{ stream_slice['extra_fields']['name'] }}",
1262
+ ],
1263
+ title="Value",
1264
+ )
1265
+ value_type: Optional[ValueType] = Field(
1266
+ None,
1267
+ description="The expected data type of the value. If omitted, the type will be inferred from the value provided.",
1268
+ title="Value Type",
1269
+ )
1270
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1271
+
1272
+
1273
+ class StreamConfig(BaseModel):
1274
+ type: Literal["StreamConfig"]
1275
+ configs_pointer: List[str] = Field(
1276
+ ...,
1277
+ description="A list of potentially nested fields indicating the full path in source config file where streams configs located.",
1278
+ examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]],
1279
+ title="Configs Pointer",
1280
+ )
1281
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1282
+
1283
+
1284
+ class ConfigComponentsResolver(BaseModel):
1285
+ type: Literal["ConfigComponentsResolver"]
1286
+ stream_config: StreamConfig
1287
+ components_mapping: List[ComponentMappingDefinition]
1288
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
754
1289
 
755
1290
 
756
1291
  class AddedFieldDefinition(BaseModel):
757
- type: Literal['AddedFieldDefinition']
1292
+ type: Literal["AddedFieldDefinition"]
758
1293
  path: List[str] = Field(
759
1294
  ...,
760
- description='List of strings defining the path where to add the value on the record.',
761
- examples=[['segment_id'], ['metadata', 'segment_id']],
762
- title='Path',
1295
+ description="List of strings defining the path where to add the value on the record.",
1296
+ examples=[["segment_id"], ["metadata", "segment_id"]],
1297
+ title="Path",
763
1298
  )
764
1299
  value: str = Field(
765
1300
  ...,
@@ -769,187 +1304,167 @@ class AddedFieldDefinition(BaseModel):
769
1304
  "{{ record['MetaData']['LastUpdatedTime'] }}",
770
1305
  "{{ stream_partition['segment_id'] }}",
771
1306
  ],
772
- title='Value',
1307
+ title="Value",
773
1308
  )
774
1309
  value_type: Optional[ValueType] = Field(
775
1310
  None,
776
- description='Type of the value. If not specified, the type will be inferred from the value.',
777
- title='Value Type',
1311
+ description="Type of the value. If not specified, the type will be inferred from the value.",
1312
+ title="Value Type",
778
1313
  )
779
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1314
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
780
1315
 
781
1316
 
782
1317
  class AddFields(BaseModel):
783
- type: Literal['AddFields']
1318
+ type: Literal["AddFields"]
784
1319
  fields: List[AddedFieldDefinition] = Field(
785
1320
  ...,
786
- description='List of transformations (path and corresponding value) that will be added to the record.',
787
- title='Fields',
1321
+ description="List of transformations (path and corresponding value) that will be added to the record.",
1322
+ title="Fields",
788
1323
  )
789
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1324
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
790
1325
 
791
1326
 
792
1327
  class ApiKeyAuthenticator(BaseModel):
793
- type: Literal['ApiKeyAuthenticator']
1328
+ type: Literal["ApiKeyAuthenticator"]
794
1329
  api_token: Optional[str] = Field(
795
1330
  None,
796
- description='The API key to inject in the request. Fill it in the user inputs.',
1331
+ description="The API key to inject in the request. Fill it in the user inputs.",
797
1332
  examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"],
798
- title='API Key',
1333
+ title="API Key",
799
1334
  )
800
1335
  header: Optional[str] = Field(
801
1336
  None,
802
- description='The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.',
803
- examples=['Authorization', 'Api-Token', 'X-Auth-Token'],
804
- title='Header Name',
1337
+ description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.",
1338
+ examples=["Authorization", "Api-Token", "X-Auth-Token"],
1339
+ title="Header Name",
805
1340
  )
806
1341
  inject_into: Optional[RequestOption] = Field(
807
1342
  None,
808
- description='Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.',
1343
+ description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.",
809
1344
  examples=[
810
- {'inject_into': 'header', 'field_name': 'Authorization'},
811
- {'inject_into': 'request_parameter', 'field_name': 'authKey'},
1345
+ {"inject_into": "header", "field_name": "Authorization"},
1346
+ {"inject_into": "request_parameter", "field_name": "authKey"},
812
1347
  ],
813
- title='Inject API Key Into Outgoing HTTP Request',
1348
+ title="Inject API Key Into Outgoing HTTP Request",
814
1349
  )
815
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1350
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
816
1351
 
817
1352
 
818
1353
  class AuthFlow(BaseModel):
819
1354
  auth_flow_type: Optional[AuthFlowType] = Field(
820
- None, description='The type of auth to use', title='Auth flow type'
1355
+ None, description="The type of auth to use", title="Auth flow type"
821
1356
  )
822
1357
  predicate_key: Optional[List[str]] = Field(
823
1358
  None,
824
- description='JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.',
825
- examples=[['credentials', 'auth_type']],
826
- title='Predicate key',
1359
+ description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.",
1360
+ examples=[["credentials", "auth_type"]],
1361
+ title="Predicate key",
827
1362
  )
828
1363
  predicate_value: Optional[str] = Field(
829
1364
  None,
830
- description='Value of the predicate_key fields for the advanced auth to be applicable.',
831
- examples=['Oauth'],
832
- title='Predicate value',
1365
+ description="Value of the predicate_key fields for the advanced auth to be applicable.",
1366
+ examples=["Oauth"],
1367
+ title="Predicate value",
833
1368
  )
834
1369
  oauth_config_specification: Optional[OAuthConfigSpecification] = None
835
1370
 
836
1371
 
837
- class CursorPagination(BaseModel):
838
- type: Literal['CursorPagination']
839
- cursor_value: str = Field(
840
- ...,
841
- description='Value of the cursor defining the next page to fetch.',
842
- examples=[
843
- '{{ headers.link.next.cursor }}',
844
- "{{ last_records[-1]['key'] }}",
845
- "{{ response['nextPage'] }}",
846
- ],
847
- title='Cursor Value',
848
- )
849
- page_size: Optional[int] = Field(
850
- None,
851
- description='The number of records to include in each pages.',
852
- examples=[100],
853
- title='Page Size',
854
- )
855
- stop_condition: Optional[str] = Field(
856
- None,
857
- description='Template string evaluating when to stop paginating.',
858
- examples=[
859
- '{{ response.data.has_more is false }}',
860
- "{{ 'next' not in headers['link'] }}",
861
- ],
862
- title='Stop Condition',
863
- )
864
- decoder: Optional[JsonDecoder] = Field(
865
- None,
866
- description='Component decoding the response so records can be extracted.',
867
- title='Decoder',
868
- )
869
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
870
-
871
-
872
1372
  class DatetimeBasedCursor(BaseModel):
873
- type: Literal['DatetimeBasedCursor']
1373
+ type: Literal["DatetimeBasedCursor"]
874
1374
  cursor_field: str = Field(
875
1375
  ...,
876
- description='The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.',
877
- examples=['created_at', "{{ config['record_cursor'] }}"],
878
- title='Cursor Field',
1376
+ description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.",
1377
+ examples=["created_at", "{{ config['record_cursor'] }}"],
1378
+ title="Cursor Field",
879
1379
  )
880
1380
  datetime_format: str = Field(
881
1381
  ...,
882
- description='The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
883
- examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s', '%ms'],
884
- title='Outgoing Datetime Format',
1382
+ description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
1383
+ examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"],
1384
+ title="Outgoing Datetime Format",
885
1385
  )
886
1386
  start_datetime: Union[str, MinMaxDatetime] = Field(
887
1387
  ...,
888
- description='The datetime that determines the earliest record that should be synced.',
889
- examples=['2020-01-1T00:00:00Z', "{{ config['start_time'] }}"],
890
- title='Start Datetime',
1388
+ description="The datetime that determines the earliest record that should be synced.",
1389
+ examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"],
1390
+ title="Start Datetime",
891
1391
  )
892
1392
  cursor_datetime_formats: Optional[List[str]] = Field(
893
1393
  None,
894
- description='The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.',
895
- title='Cursor Datetime Formats',
1394
+ description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.",
1395
+ title="Cursor Datetime Formats",
896
1396
  )
897
1397
  cursor_granularity: Optional[str] = Field(
898
1398
  None,
899
- description='Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.',
900
- examples=['PT1S'],
901
- title='Cursor Granularity',
1399
+ description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.",
1400
+ examples=["PT1S"],
1401
+ title="Cursor Granularity",
902
1402
  )
903
1403
  end_datetime: Optional[Union[str, MinMaxDatetime]] = Field(
904
1404
  None,
905
- description='The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.',
906
- examples=['2021-01-1T00:00:00Z', '{{ now_utc() }}', '{{ day_delta(-1) }}'],
907
- title='End Datetime',
1405
+ description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.",
1406
+ examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"],
1407
+ title="End Datetime",
908
1408
  )
909
1409
  end_time_option: Optional[RequestOption] = Field(
910
1410
  None,
911
- description='Optionally configures how the end datetime will be sent in requests to the source API.',
912
- title='Inject End Time Into Outgoing HTTP Request',
1411
+ description="Optionally configures how the end datetime will be sent in requests to the source API.",
1412
+ title="Inject End Time Into Outgoing HTTP Request",
913
1413
  )
914
1414
  is_data_feed: Optional[bool] = Field(
915
1415
  None,
916
- description='A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.',
917
- title='Whether the target API is formatted as a data feed',
1416
+ description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.",
1417
+ title="Whether the target API is formatted as a data feed",
1418
+ )
1419
+ is_client_side_incremental: Optional[bool] = Field(
1420
+ None,
1421
+ description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.",
1422
+ title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)",
1423
+ )
1424
+ is_compare_strictly: Optional[bool] = Field(
1425
+ False,
1426
+ description="Set to True if the target API does not accept queries where the start time equal the end time.",
1427
+ title="Whether to skip requests if the start time equals the end time",
1428
+ )
1429
+ global_substream_cursor: Optional[bool] = Field(
1430
+ False,
1431
+ description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).",
1432
+ title="Whether to store cursor as one value instead of per partition",
918
1433
  )
919
1434
  lookback_window: Optional[str] = Field(
920
1435
  None,
921
- description='Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.',
922
- examples=['P1D', "P{{ config['lookback_days'] }}D"],
923
- title='Lookback Window',
1436
+ description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.",
1437
+ examples=["P1D", "P{{ config['lookback_days'] }}D"],
1438
+ title="Lookback Window",
924
1439
  )
925
1440
  partition_field_end: Optional[str] = Field(
926
1441
  None,
927
- description='Name of the partition start time field.',
928
- examples=['ending_time'],
929
- title='Partition Field End',
1442
+ description="Name of the partition start time field.",
1443
+ examples=["ending_time"],
1444
+ title="Partition Field End",
930
1445
  )
931
1446
  partition_field_start: Optional[str] = Field(
932
1447
  None,
933
- description='Name of the partition end time field.',
934
- examples=['starting_time'],
935
- title='Partition Field Start',
1448
+ description="Name of the partition end time field.",
1449
+ examples=["starting_time"],
1450
+ title="Partition Field Start",
936
1451
  )
937
1452
  start_time_option: Optional[RequestOption] = Field(
938
1453
  None,
939
- description='Optionally configures how the start datetime will be sent in requests to the source API.',
940
- title='Inject Start Time Into Outgoing HTTP Request',
1454
+ description="Optionally configures how the start datetime will be sent in requests to the source API.",
1455
+ title="Inject Start Time Into Outgoing HTTP Request",
941
1456
  )
942
1457
  step: Optional[str] = Field(
943
1458
  None,
944
- description='The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.',
945
- examples=['P1W', "{{ config['step_increment'] }}"],
946
- title='Step',
1459
+ description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.",
1460
+ examples=["P1W", "{{ config['step_increment'] }}"],
1461
+ title="Step",
947
1462
  )
948
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1463
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
949
1464
 
950
1465
 
951
1466
  class DefaultErrorHandler(BaseModel):
952
- type: Literal['DefaultErrorHandler']
1467
+ type: Literal["DefaultErrorHandler"]
953
1468
  backoff_strategies: Optional[
954
1469
  List[
955
1470
  Union[
@@ -962,154 +1477,188 @@ class DefaultErrorHandler(BaseModel):
962
1477
  ]
963
1478
  ] = Field(
964
1479
  None,
965
- description='List of backoff strategies to use to determine how long to wait before retrying a retryable request.',
966
- title='Backoff Strategies',
1480
+ description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.",
1481
+ title="Backoff Strategies",
967
1482
  )
968
1483
  max_retries: Optional[int] = Field(
969
1484
  5,
970
- description='The maximum number of time to retry a retryable request before giving up and failing.',
1485
+ description="The maximum number of time to retry a retryable request before giving up and failing.",
971
1486
  examples=[5, 0, 10],
972
- title='Max Retry Count',
1487
+ title="Max Retry Count",
973
1488
  )
974
1489
  response_filters: Optional[List[HttpResponseFilter]] = Field(
975
1490
  None,
976
1491
  description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.",
977
- title='Response Filters',
1492
+ title="Response Filters",
978
1493
  )
979
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1494
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
980
1495
 
981
1496
 
982
1497
  class DefaultPaginator(BaseModel):
983
- type: Literal['DefaultPaginator']
1498
+ type: Literal["DefaultPaginator"]
984
1499
  pagination_strategy: Union[
985
1500
  CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement
986
1501
  ] = Field(
987
1502
  ...,
988
- description='Strategy defining how records are paginated.',
989
- title='Pagination Strategy',
990
- )
991
- decoder: Optional[JsonDecoder] = Field(
992
- None,
993
- description='Component decoding the response so records can be extracted.',
994
- title='Decoder',
1503
+ description="Strategy defining how records are paginated.",
1504
+ title="Pagination Strategy",
995
1505
  )
996
1506
  page_size_option: Optional[RequestOption] = None
997
1507
  page_token_option: Optional[Union[RequestOption, RequestPath]] = None
998
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
999
-
1000
-
1001
- class DpathExtractor(BaseModel):
1002
- type: Literal['DpathExtractor']
1003
- field_path: List[str] = Field(
1004
- ...,
1005
- description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
1006
- examples=[
1007
- ['data'],
1008
- ['data', 'records'],
1009
- ['data', '{{ parameters.name }}'],
1010
- ['data', '*', 'record'],
1011
- ],
1012
- title='Field Path',
1013
- )
1014
- decoder: Optional[JsonDecoder] = Field(
1015
- None,
1016
- description='Component decoding the response so records can be extracted.',
1017
- title='Decoder',
1018
- )
1019
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1508
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1020
1509
 
1021
1510
 
1022
1511
  class SessionTokenRequestApiKeyAuthenticator(BaseModel):
1023
- type: Literal['ApiKey']
1512
+ type: Literal["ApiKey"]
1024
1513
  inject_into: RequestOption = Field(
1025
1514
  ...,
1026
- description='Configure how the API Key will be sent in requests to the source API.',
1515
+ description="Configure how the API Key will be sent in requests to the source API.",
1027
1516
  examples=[
1028
- {'inject_into': 'header', 'field_name': 'Authorization'},
1029
- {'inject_into': 'request_parameter', 'field_name': 'authKey'},
1517
+ {"inject_into": "header", "field_name": "Authorization"},
1518
+ {"inject_into": "request_parameter", "field_name": "authKey"},
1030
1519
  ],
1031
- title='Inject API Key Into Outgoing HTTP Request',
1520
+ title="Inject API Key Into Outgoing HTTP Request",
1032
1521
  )
1033
1522
 
1034
1523
 
1035
1524
  class ListPartitionRouter(BaseModel):
1036
- type: Literal['ListPartitionRouter']
1525
+ type: Literal["ListPartitionRouter"]
1037
1526
  cursor_field: str = Field(
1038
1527
  ...,
1039
1528
  description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.',
1040
- examples=['section', "{{ config['section_key'] }}"],
1041
- title='Current Partition Value Identifier',
1529
+ examples=["section", "{{ config['section_key'] }}"],
1530
+ title="Current Partition Value Identifier",
1042
1531
  )
1043
1532
  values: Union[str, List[str]] = Field(
1044
1533
  ...,
1045
- description='The list of attributes being iterated over and used as input for the requests made to the source API.',
1046
- examples=[['section_a', 'section_b', 'section_c'], "{{ config['sections'] }}"],
1047
- title='Partition Values',
1534
+ description="The list of attributes being iterated over and used as input for the requests made to the source API.",
1535
+ examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"],
1536
+ title="Partition Values",
1048
1537
  )
1049
1538
  request_option: Optional[RequestOption] = Field(
1050
1539
  None,
1051
- description='A request option describing where the list value should be injected into and under what field name if applicable.',
1052
- title='Inject Partition Value Into Outgoing HTTP Request',
1540
+ description="A request option describing where the list value should be injected into and under what field name if applicable.",
1541
+ title="Inject Partition Value Into Outgoing HTTP Request",
1053
1542
  )
1054
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1543
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1055
1544
 
1056
1545
 
1057
1546
  class RecordSelector(BaseModel):
1058
- type: Literal['RecordSelector']
1547
+ type: Literal["RecordSelector"]
1059
1548
  extractor: Union[CustomRecordExtractor, DpathExtractor]
1060
1549
  record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field(
1061
1550
  None,
1062
- description='Responsible for filtering records to be emitted by the Source.',
1063
- title='Record Filter',
1551
+ description="Responsible for filtering records to be emitted by the Source.",
1552
+ title="Record Filter",
1553
+ )
1554
+ schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field(
1555
+ SchemaNormalization.None_,
1556
+ description="Responsible for normalization according to the schema.",
1557
+ title="Schema Normalization",
1064
1558
  )
1065
- schema_normalization: Optional[SchemaNormalization] = SchemaNormalization.None_
1066
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1559
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1560
+
1561
+
1562
+ class GzipParser(BaseModel):
1563
+ type: Literal["GzipParser"]
1564
+ inner_parser: Union[JsonLineParser, CsvParser]
1067
1565
 
1068
1566
 
1069
1567
  class Spec(BaseModel):
1070
- type: Literal['Spec']
1568
+ type: Literal["Spec"]
1071
1569
  connection_specification: Dict[str, Any] = Field(
1072
1570
  ...,
1073
- description='A connection specification describing how a the connector can be configured.',
1074
- title='Connection Specification',
1571
+ description="A connection specification describing how a the connector can be configured.",
1572
+ title="Connection Specification",
1075
1573
  )
1076
1574
  documentation_url: Optional[str] = Field(
1077
1575
  None,
1078
1576
  description="URL of the connector's documentation page.",
1079
- examples=['https://docs.airbyte.com/integrations/sources/dremio'],
1080
- title='Documentation URL',
1577
+ examples=["https://docs.airbyte.com/integrations/sources/dremio"],
1578
+ title="Documentation URL",
1081
1579
  )
1082
1580
  advanced_auth: Optional[AuthFlow] = Field(
1083
1581
  None,
1084
- description='Advanced specification for configuring the authentication flow.',
1085
- title='Advanced Auth',
1582
+ description="Advanced specification for configuring the authentication flow.",
1583
+ title="Advanced Auth",
1086
1584
  )
1087
1585
 
1088
1586
 
1089
1587
  class CompositeErrorHandler(BaseModel):
1090
- type: Literal['CompositeErrorHandler']
1588
+ type: Literal["CompositeErrorHandler"]
1091
1589
  error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field(
1092
1590
  ...,
1093
- description='List of error handlers to iterate on to determine how to handle a failed response.',
1094
- title='Error Handlers',
1591
+ description="List of error handlers to iterate on to determine how to handle a failed response.",
1592
+ title="Error Handlers",
1095
1593
  )
1096
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1594
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1097
1595
 
1098
1596
 
1099
- class DeclarativeSource(BaseModel):
1597
+ class CompositeRawDecoder(BaseModel):
1598
+ type: Literal["CompositeRawDecoder"]
1599
+ parser: Union[GzipParser, JsonLineParser, CsvParser]
1600
+
1601
+
1602
+ class DeclarativeSource1(BaseModel):
1100
1603
  class Config:
1101
1604
  extra = Extra.forbid
1102
1605
 
1103
- type: Literal['DeclarativeSource']
1606
+ type: Literal["DeclarativeSource"]
1104
1607
  check: CheckStream
1105
1608
  streams: List[DeclarativeStream]
1106
- version: str
1609
+ dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None
1610
+ version: str = Field(
1611
+ ...,
1612
+ description="The version of the Airbyte CDK used to build and test the source.",
1613
+ )
1614
+ schemas: Optional[Schemas] = None
1615
+ definitions: Optional[Dict[str, Any]] = None
1616
+ spec: Optional[Spec] = None
1617
+ concurrency_level: Optional[ConcurrencyLevel] = None
1618
+ metadata: Optional[Dict[str, Any]] = Field(
1619
+ None,
1620
+ description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
1621
+ )
1622
+ description: Optional[str] = Field(
1623
+ None,
1624
+ description="A description of the connector. It will be presented on the Source documentation page.",
1625
+ )
1626
+
1627
+
1628
+ class DeclarativeSource2(BaseModel):
1629
+ class Config:
1630
+ extra = Extra.forbid
1631
+
1632
+ type: Literal["DeclarativeSource"]
1633
+ check: CheckStream
1634
+ streams: Optional[List[DeclarativeStream]] = None
1635
+ dynamic_streams: List[DynamicDeclarativeStream]
1636
+ version: str = Field(
1637
+ ...,
1638
+ description="The version of the Airbyte CDK used to build and test the source.",
1639
+ )
1107
1640
  schemas: Optional[Schemas] = None
1108
1641
  definitions: Optional[Dict[str, Any]] = None
1109
1642
  spec: Optional[Spec] = None
1643
+ concurrency_level: Optional[ConcurrencyLevel] = None
1110
1644
  metadata: Optional[Dict[str, Any]] = Field(
1111
1645
  None,
1112
- description='For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.',
1646
+ description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
1647
+ )
1648
+ description: Optional[str] = Field(
1649
+ None,
1650
+ description="A description of the connector. It will be presented on the Source documentation page.",
1651
+ )
1652
+
1653
+
1654
+ class DeclarativeSource(BaseModel):
1655
+ class Config:
1656
+ extra = Extra.forbid
1657
+
1658
+ __root__: Union[DeclarativeSource1, DeclarativeSource2] = Field(
1659
+ ...,
1660
+ description="An API source that extracts data according to its declarative components.",
1661
+ title="DeclarativeSource",
1113
1662
  )
1114
1663
 
1115
1664
 
@@ -1117,12 +1666,12 @@ class SelectiveAuthenticator(BaseModel):
1117
1666
  class Config:
1118
1667
  extra = Extra.allow
1119
1668
 
1120
- type: Literal['SelectiveAuthenticator']
1669
+ type: Literal["SelectiveAuthenticator"]
1121
1670
  authenticator_selection_path: List[str] = Field(
1122
1671
  ...,
1123
- description='Path of the field in config with selected authenticator name',
1124
- examples=[['auth'], ['auth', 'type']],
1125
- title='Authenticator Selection Path',
1672
+ description="Path of the field in config with selected authenticator name",
1673
+ examples=[["auth"], ["auth", "type"]],
1674
+ title="Authenticator Selection Path",
1126
1675
  )
1127
1676
  authenticators: Dict[
1128
1677
  str,
@@ -1132,127 +1681,150 @@ class SelectiveAuthenticator(BaseModel):
1132
1681
  BearerAuthenticator,
1133
1682
  CustomAuthenticator,
1134
1683
  OAuthAuthenticator,
1684
+ JwtAuthenticator,
1135
1685
  NoAuth,
1136
1686
  SessionTokenAuthenticator,
1137
1687
  LegacySessionTokenAuthenticator,
1138
1688
  ],
1139
1689
  ] = Field(
1140
1690
  ...,
1141
- description='Authenticators to select from.',
1691
+ description="Authenticators to select from.",
1142
1692
  examples=[
1143
1693
  {
1144
- 'authenticators': {
1145
- 'token': '#/definitions/ApiKeyAuthenticator',
1146
- 'oauth': '#/definitions/OAuthAuthenticator',
1694
+ "authenticators": {
1695
+ "token": "#/definitions/ApiKeyAuthenticator",
1696
+ "oauth": "#/definitions/OAuthAuthenticator",
1697
+ "jwt": "#/definitions/JwtAuthenticator",
1147
1698
  }
1148
1699
  }
1149
1700
  ],
1150
- title='Authenticators',
1701
+ title="Authenticators",
1151
1702
  )
1152
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1703
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1153
1704
 
1154
1705
 
1155
1706
  class DeclarativeStream(BaseModel):
1156
1707
  class Config:
1157
1708
  extra = Extra.allow
1158
1709
 
1159
- type: Literal['DeclarativeStream']
1160
- retriever: Union[CustomRetriever, SimpleRetriever] = Field(
1710
+ type: Literal["DeclarativeStream"]
1711
+ retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
1161
1712
  ...,
1162
- description='Component used to coordinate how records are extracted across stream slices and request pages.',
1163
- title='Retriever',
1713
+ description="Component used to coordinate how records are extracted across stream slices and request pages.",
1714
+ title="Retriever",
1164
1715
  )
1165
- incremental_sync: Optional[
1166
- Union[CustomIncrementalSync, DatetimeBasedCursor]
1167
- ] = Field(
1716
+ incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field(
1168
1717
  None,
1169
- description='Component used to fetch data incrementally based on a time field in the data.',
1170
- title='Incremental Sync',
1171
- )
1172
- name: Optional[str] = Field(
1173
- '', description='The stream name.', example=['Users'], title='Name'
1718
+ description="Component used to fetch data incrementally based on a time field in the data.",
1719
+ title="Incremental Sync",
1174
1720
  )
1721
+ name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name")
1175
1722
  primary_key: Optional[PrimaryKey] = Field(
1176
- '', description='The primary key of the stream.', title='Primary Key'
1723
+ "", description="The primary key of the stream.", title="Primary Key"
1177
1724
  )
1178
1725
  schema_loader: Optional[
1179
- Union[InlineSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]
1726
+ Union[
1727
+ DynamicSchemaLoader,
1728
+ InlineSchemaLoader,
1729
+ JsonFileSchemaLoader,
1730
+ CustomSchemaLoader,
1731
+ ]
1180
1732
  ] = Field(
1181
1733
  None,
1182
- description='Component used to retrieve the schema for the current stream.',
1183
- title='Schema Loader',
1734
+ description="Component used to retrieve the schema for the current stream.",
1735
+ title="Schema Loader",
1184
1736
  )
1185
1737
  transformations: Optional[
1186
- List[Union[AddFields, CustomTransformation, RemoveFields]]
1738
+ List[
1739
+ Union[
1740
+ AddFields,
1741
+ CustomTransformation,
1742
+ RemoveFields,
1743
+ KeysToLower,
1744
+ KeysToSnakeCase,
1745
+ FlattenFields,
1746
+ KeysReplace,
1747
+ ]
1748
+ ]
1187
1749
  ] = Field(
1188
1750
  None,
1189
- description='A list of transformations to be applied to each output record.',
1190
- title='Transformations',
1751
+ description="A list of transformations to be applied to each output record.",
1752
+ title="Transformations",
1191
1753
  )
1192
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1754
+ state_migrations: Optional[
1755
+ List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]
1756
+ ] = Field(
1757
+ [],
1758
+ description="Array of state migrations to be applied on the input state",
1759
+ title="State Migrations",
1760
+ )
1761
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1193
1762
 
1194
1763
 
1195
1764
  class SessionTokenAuthenticator(BaseModel):
1196
- type: Literal['SessionTokenAuthenticator']
1765
+ type: Literal["SessionTokenAuthenticator"]
1197
1766
  login_requester: HttpRequester = Field(
1198
1767
  ...,
1199
- description='Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.',
1768
+ description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.",
1200
1769
  examples=[
1201
1770
  {
1202
- 'type': 'HttpRequester',
1203
- 'url_base': 'https://my_api.com',
1204
- 'path': '/login',
1205
- 'authenticator': {
1206
- 'type': 'BasicHttpAuthenticator',
1207
- 'username': '{{ config.username }}',
1208
- 'password': '{{ config.password }}',
1771
+ "type": "HttpRequester",
1772
+ "url_base": "https://my_api.com",
1773
+ "path": "/login",
1774
+ "authenticator": {
1775
+ "type": "BasicHttpAuthenticator",
1776
+ "username": "{{ config.username }}",
1777
+ "password": "{{ config.password }}",
1209
1778
  },
1210
1779
  }
1211
1780
  ],
1212
- title='Login Requester',
1781
+ title="Login Requester",
1213
1782
  )
1214
1783
  session_token_path: List[str] = Field(
1215
1784
  ...,
1216
- description='The path in the response body returned from the login requester to the session token.',
1217
- examples=[['access_token'], ['result', 'token']],
1218
- title='Session Token Path',
1785
+ description="The path in the response body returned from the login requester to the session token.",
1786
+ examples=[["access_token"], ["result", "token"]],
1787
+ title="Session Token Path",
1219
1788
  )
1220
1789
  expiration_duration: Optional[str] = Field(
1221
1790
  None,
1222
- description='The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.',
1223
- examples=['PT1H', 'P1D'],
1224
- title='Expiration Duration',
1791
+ description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.",
1792
+ examples=["PT1H", "P1D"],
1793
+ title="Expiration Duration",
1225
1794
  )
1226
1795
  request_authentication: Union[
1227
1796
  SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator
1228
1797
  ] = Field(
1229
1798
  ...,
1230
- description='Authentication method to use for requests sent to the API, specifying how to inject the session token.',
1231
- title='Data Request Authentication',
1799
+ description="Authentication method to use for requests sent to the API, specifying how to inject the session token.",
1800
+ title="Data Request Authentication",
1232
1801
  )
1233
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1802
+ decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field(
1803
+ None, description="Component used to decode the response.", title="Decoder"
1804
+ )
1805
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1234
1806
 
1235
1807
 
1236
1808
  class HttpRequester(BaseModel):
1237
- type: Literal['HttpRequester']
1809
+ type: Literal["HttpRequester"]
1238
1810
  url_base: str = Field(
1239
1811
  ...,
1240
- description='Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.',
1812
+ description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
1241
1813
  examples=[
1242
- 'https://connect.squareup.com/v2',
1814
+ "https://connect.squareup.com/v2",
1243
1815
  "{{ config['base_url'] or 'https://app.posthog.com'}}/api/",
1244
1816
  ],
1245
- title='API Base URL',
1817
+ title="API Base URL",
1246
1818
  )
1247
1819
  path: str = Field(
1248
1820
  ...,
1249
- description='Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.',
1821
+ description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
1250
1822
  examples=[
1251
- '/products',
1823
+ "/products",
1252
1824
  "/quotes/{{ stream_partition['id'] }}/quote_line_groups",
1253
1825
  "/trades/{{ config['symbol_id'] }}/history",
1254
1826
  ],
1255
- title='URL Path',
1827
+ title="URL Path",
1256
1828
  )
1257
1829
  authenticator: Optional[
1258
1830
  Union[
@@ -1261,6 +1833,7 @@ class HttpRequester(BaseModel):
1261
1833
  BearerAuthenticator,
1262
1834
  CustomAuthenticator,
1263
1835
  OAuthAuthenticator,
1836
+ JwtAuthenticator,
1264
1837
  NoAuth,
1265
1838
  SessionTokenAuthenticator,
1266
1839
  LegacySessionTokenAuthenticator,
@@ -1268,101 +1841,139 @@ class HttpRequester(BaseModel):
1268
1841
  ]
1269
1842
  ] = Field(
1270
1843
  None,
1271
- description='Authentication method to use for requests sent to the API.',
1272
- title='Authenticator',
1844
+ description="Authentication method to use for requests sent to the API.",
1845
+ title="Authenticator",
1273
1846
  )
1274
1847
  error_handler: Optional[
1275
1848
  Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]
1276
1849
  ] = Field(
1277
1850
  None,
1278
- description='Error handler component that defines how to handle errors.',
1279
- title='Error Handler',
1851
+ description="Error handler component that defines how to handle errors.",
1852
+ title="Error Handler",
1280
1853
  )
1281
1854
  http_method: Optional[HttpMethod] = Field(
1282
1855
  HttpMethod.GET,
1283
- description='The HTTP method used to fetch data from the source (can be GET or POST).',
1284
- examples=['GET', 'POST'],
1285
- title='HTTP Method',
1856
+ description="The HTTP method used to fetch data from the source (can be GET or POST).",
1857
+ examples=["GET", "POST"],
1858
+ title="HTTP Method",
1286
1859
  )
1287
1860
  request_body_data: Optional[Union[str, Dict[str, str]]] = Field(
1288
1861
  None,
1289
- description='Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.',
1862
+ description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.",
1290
1863
  examples=[
1291
1864
  '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n'
1292
1865
  ],
1293
- title='Request Body Payload (Non-JSON)',
1866
+ title="Request Body Payload (Non-JSON)",
1294
1867
  )
1295
1868
  request_body_json: Optional[Union[str, Dict[str, Any]]] = Field(
1296
1869
  None,
1297
- description='Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.',
1870
+ description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.",
1298
1871
  examples=[
1299
- {'sort_order': 'ASC', 'sort_field': 'CREATED_AT'},
1300
- {'key': "{{ config['value'] }}"},
1301
- {'sort': {'field': 'updated_at', 'order': 'ascending'}},
1872
+ {"sort_order": "ASC", "sort_field": "CREATED_AT"},
1873
+ {"key": "{{ config['value'] }}"},
1874
+ {"sort": {"field": "updated_at", "order": "ascending"}},
1302
1875
  ],
1303
- title='Request Body JSON Payload',
1876
+ title="Request Body JSON Payload",
1304
1877
  )
1305
1878
  request_headers: Optional[Union[str, Dict[str, str]]] = Field(
1306
1879
  None,
1307
- description='Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.',
1308
- examples=[{'Output-Format': 'JSON'}, {'Version': "{{ config['version'] }}"}],
1309
- title='Request Headers',
1880
+ description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.",
1881
+ examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
1882
+ title="Request Headers",
1310
1883
  )
1311
1884
  request_parameters: Optional[Union[str, Dict[str, str]]] = Field(
1312
1885
  None,
1313
- description='Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.',
1886
+ description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
1314
1887
  examples=[
1315
- {'unit': 'day'},
1888
+ {"unit": "day"},
1316
1889
  {
1317
- 'query': 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
1890
+ "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
1318
1891
  },
1319
- {'searchIn': "{{ ','.join(config.get('search_in', [])) }}"},
1320
- {'sort_by[asc]': 'updated_at'},
1892
+ {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
1893
+ {"sort_by[asc]": "updated_at"},
1321
1894
  ],
1322
- title='Query Parameters',
1895
+ title="Query Parameters",
1323
1896
  )
1324
1897
  use_cache: Optional[bool] = Field(
1325
1898
  False,
1326
- description='Enables stream requests caching. This field is automatically set by the CDK.',
1327
- title='Use Cache',
1899
+ description="Enables stream requests caching. This field is automatically set by the CDK.",
1900
+ title="Use Cache",
1901
+ )
1902
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1903
+
1904
+
1905
+ class DynamicSchemaLoader(BaseModel):
1906
+ type: Literal["DynamicSchemaLoader"]
1907
+ retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
1908
+ ...,
1909
+ description="Component used to coordinate how records are extracted across stream slices and request pages.",
1910
+ title="Retriever",
1911
+ )
1912
+ schema_transformations: Optional[
1913
+ List[
1914
+ Union[
1915
+ AddFields,
1916
+ CustomTransformation,
1917
+ RemoveFields,
1918
+ KeysToLower,
1919
+ KeysToSnakeCase,
1920
+ FlattenFields,
1921
+ KeysReplace,
1922
+ ]
1923
+ ]
1924
+ ] = Field(
1925
+ None,
1926
+ description="A list of transformations to be applied to the schema.",
1927
+ title="Schema Transformations",
1328
1928
  )
1329
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1929
+ schema_type_identifier: SchemaTypeIdentifier
1930
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1330
1931
 
1331
1932
 
1332
1933
  class ParentStreamConfig(BaseModel):
1333
- type: Literal['ParentStreamConfig']
1934
+ type: Literal["ParentStreamConfig"]
1334
1935
  parent_key: str = Field(
1335
1936
  ...,
1336
- description='The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.',
1337
- examples=['id', "{{ config['parent_record_id'] }}"],
1338
- title='Parent Key',
1937
+ description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.",
1938
+ examples=["id", "{{ config['parent_record_id'] }}"],
1939
+ title="Parent Key",
1339
1940
  )
1340
1941
  stream: DeclarativeStream = Field(
1341
- ..., description='Reference to the parent stream.', title='Parent Stream'
1942
+ ..., description="Reference to the parent stream.", title="Parent Stream"
1342
1943
  )
1343
1944
  partition_field: str = Field(
1344
1945
  ...,
1345
- description='While iterating over parent records during a sync, the parent_key value can be referenced by using this field.',
1346
- examples=['parent_id', "{{ config['parent_partition_field'] }}"],
1347
- title='Current Parent Key Value Identifier',
1946
+ description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.",
1947
+ examples=["parent_id", "{{ config['parent_partition_field'] }}"],
1948
+ title="Current Parent Key Value Identifier",
1348
1949
  )
1349
1950
  request_option: Optional[RequestOption] = Field(
1350
1951
  None,
1351
- description='A request option describing where the parent key value should be injected into and under what field name if applicable.',
1352
- title='Request Option',
1952
+ description="A request option describing where the parent key value should be injected into and under what field name if applicable.",
1953
+ title="Request Option",
1954
+ )
1955
+ incremental_dependency: Optional[bool] = Field(
1956
+ False,
1957
+ description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.",
1958
+ title="Incremental Dependency",
1959
+ )
1960
+ extra_fields: Optional[List[List[str]]] = Field(
1961
+ None,
1962
+ description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.",
1963
+ title="Extra Fields",
1353
1964
  )
1354
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1965
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1355
1966
 
1356
1967
 
1357
1968
  class SimpleRetriever(BaseModel):
1358
- type: Literal['SimpleRetriever']
1969
+ type: Literal["SimpleRetriever"]
1359
1970
  record_selector: RecordSelector = Field(
1360
1971
  ...,
1361
- description='Component that describes how to extract records from a HTTP response.',
1972
+ description="Component that describes how to extract records from a HTTP response.",
1362
1973
  )
1363
1974
  requester: Union[CustomRequester, HttpRequester] = Field(
1364
1975
  ...,
1365
- description='Requester component that describes how to prepare HTTP requests to send to the source API.',
1976
+ description="Requester component that describes how to prepare HTTP requests to send to the source API.",
1366
1977
  )
1367
1978
  paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
1368
1979
  None,
@@ -1370,40 +1981,163 @@ class SimpleRetriever(BaseModel):
1370
1981
  )
1371
1982
  ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field(
1372
1983
  False,
1373
- description='If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.',
1984
+ description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.",
1985
+ )
1986
+ partition_router: Optional[
1987
+ Union[
1988
+ CustomPartitionRouter,
1989
+ ListPartitionRouter,
1990
+ SubstreamPartitionRouter,
1991
+ List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
1992
+ ]
1993
+ ] = Field(
1994
+ [],
1995
+ description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
1996
+ title="Partition Router",
1997
+ )
1998
+ decoder: Optional[
1999
+ Union[
2000
+ CustomDecoder,
2001
+ JsonDecoder,
2002
+ JsonlDecoder,
2003
+ IterableDecoder,
2004
+ XmlDecoder,
2005
+ GzipJsonDecoder,
2006
+ CompositeRawDecoder,
2007
+ ]
2008
+ ] = Field(
2009
+ None,
2010
+ description="Component decoding the response so records can be extracted.",
2011
+ title="Decoder",
2012
+ )
2013
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
2014
+
2015
+
2016
+ class AsyncRetriever(BaseModel):
2017
+ type: Literal["AsyncRetriever"]
2018
+ record_selector: RecordSelector = Field(
2019
+ ...,
2020
+ description="Component that describes how to extract records from a HTTP response.",
2021
+ )
2022
+ status_mapping: AsyncJobStatusMap = Field(
2023
+ ..., description="Async Job Status to Airbyte CDK Async Job Status mapping."
2024
+ )
2025
+ status_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
2026
+ ..., description="Responsible for fetching the actual status of the async job."
2027
+ )
2028
+ urls_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
2029
+ ...,
2030
+ description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.",
2031
+ )
2032
+ download_extractor: Optional[
2033
+ Union[CustomRecordExtractor, DpathExtractor, ResponseToFileExtractor]
2034
+ ] = Field(None, description="Responsible for fetching the records from provided urls.")
2035
+ creation_requester: Union[CustomRequester, HttpRequester] = Field(
2036
+ ...,
2037
+ description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.",
2038
+ )
2039
+ polling_requester: Union[CustomRequester, HttpRequester] = Field(
2040
+ ...,
2041
+ description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.",
2042
+ )
2043
+ download_requester: Union[CustomRequester, HttpRequester] = Field(
2044
+ ...,
2045
+ description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.",
2046
+ )
2047
+ download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
2048
+ None,
2049
+ description="Paginator component that describes how to navigate through the API's pages during download.",
2050
+ )
2051
+ abort_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
2052
+ None,
2053
+ description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.",
2054
+ )
2055
+ delete_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
2056
+ None,
2057
+ description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.",
1374
2058
  )
1375
2059
  partition_router: Optional[
1376
2060
  Union[
1377
2061
  CustomPartitionRouter,
1378
2062
  ListPartitionRouter,
1379
2063
  SubstreamPartitionRouter,
1380
- List[
1381
- Union[
1382
- CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
1383
- ]
1384
- ],
2064
+ List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
1385
2065
  ]
1386
2066
  ] = Field(
1387
2067
  [],
1388
- description='PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.',
1389
- title='Partition Router',
2068
+ description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
2069
+ title="Partition Router",
2070
+ )
2071
+ decoder: Optional[
2072
+ Union[
2073
+ CustomDecoder,
2074
+ JsonDecoder,
2075
+ JsonlDecoder,
2076
+ IterableDecoder,
2077
+ XmlDecoder,
2078
+ GzipJsonDecoder,
2079
+ ]
2080
+ ] = Field(
2081
+ None,
2082
+ description="Component decoding the response so records can be extracted.",
2083
+ title="Decoder",
2084
+ )
2085
+ download_decoder: Optional[
2086
+ Union[
2087
+ CustomDecoder,
2088
+ JsonDecoder,
2089
+ JsonlDecoder,
2090
+ IterableDecoder,
2091
+ XmlDecoder,
2092
+ GzipJsonDecoder,
2093
+ ]
2094
+ ] = Field(
2095
+ None,
2096
+ description="Component decoding the download response so records can be extracted.",
2097
+ title="Download Decoder",
1390
2098
  )
1391
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
2099
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
1392
2100
 
1393
2101
 
1394
2102
  class SubstreamPartitionRouter(BaseModel):
1395
- type: Literal['SubstreamPartitionRouter']
2103
+ type: Literal["SubstreamPartitionRouter"]
1396
2104
  parent_stream_configs: List[ParentStreamConfig] = Field(
1397
2105
  ...,
1398
- description='Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.',
1399
- title='Parent Stream Configs',
2106
+ description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.",
2107
+ title="Parent Stream Configs",
2108
+ )
2109
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
2110
+
2111
+
2112
+ class HttpComponentsResolver(BaseModel):
2113
+ type: Literal["HttpComponentsResolver"]
2114
+ retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
2115
+ ...,
2116
+ description="Component used to coordinate how records are extracted across stream slices and request pages.",
2117
+ title="Retriever",
2118
+ )
2119
+ components_mapping: List[ComponentMappingDefinition]
2120
+ parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
2121
+
2122
+
2123
+ class DynamicDeclarativeStream(BaseModel):
2124
+ type: Literal["DynamicDeclarativeStream"]
2125
+ stream_template: DeclarativeStream = Field(
2126
+ ..., description="Reference to the stream template.", title="Stream Template"
2127
+ )
2128
+ components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field(
2129
+ ...,
2130
+ description="Component resolve and populates stream templates with components values.",
2131
+ title="Components Resolver",
1400
2132
  )
1401
- parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
1402
2133
 
1403
2134
 
1404
2135
  CompositeErrorHandler.update_forward_refs()
1405
- DeclarativeSource.update_forward_refs()
2136
+ DeclarativeSource1.update_forward_refs()
2137
+ DeclarativeSource2.update_forward_refs()
1406
2138
  SelectiveAuthenticator.update_forward_refs()
1407
2139
  DeclarativeStream.update_forward_refs()
1408
2140
  SessionTokenAuthenticator.update_forward_refs()
2141
+ DynamicSchemaLoader.update_forward_refs()
1409
2142
  SimpleRetriever.update_forward_refs()
2143
+ AsyncRetriever.update_forward_refs()