airbyte-cdk 0.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (368) hide show
  1. airbyte_cdk/__init__.py +358 -0
  2. airbyte_cdk/cli/__init__.py +1 -0
  3. airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
  4. airbyte_cdk/cli/source_declarative_manifest/_run.py +236 -0
  5. airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
  6. airbyte_cdk/config_observation.py +104 -0
  7. airbyte_cdk/connector.py +123 -0
  8. airbyte_cdk/connector_builder/README.md +53 -0
  9. airbyte_cdk/connector_builder/__init__.py +3 -0
  10. airbyte_cdk/connector_builder/connector_builder_handler.py +121 -0
  11. airbyte_cdk/connector_builder/main.py +107 -0
  12. airbyte_cdk/connector_builder/models.py +73 -0
  13. airbyte_cdk/connector_builder/test_reader/__init__.py +7 -0
  14. airbyte_cdk/connector_builder/test_reader/helpers.py +689 -0
  15. airbyte_cdk/connector_builder/test_reader/message_grouper.py +173 -0
  16. airbyte_cdk/connector_builder/test_reader/reader.py +441 -0
  17. airbyte_cdk/connector_builder/test_reader/types.py +83 -0
  18. airbyte_cdk/destinations/__init__.py +8 -0
  19. airbyte_cdk/destinations/destination.py +154 -0
  20. airbyte_cdk/destinations/vector_db_based/README.md +37 -0
  21. airbyte_cdk/destinations/vector_db_based/__init__.py +38 -0
  22. airbyte_cdk/destinations/vector_db_based/config.py +298 -0
  23. airbyte_cdk/destinations/vector_db_based/document_processor.py +223 -0
  24. airbyte_cdk/destinations/vector_db_based/embedder.py +303 -0
  25. airbyte_cdk/destinations/vector_db_based/indexer.py +78 -0
  26. airbyte_cdk/destinations/vector_db_based/test_utils.py +63 -0
  27. airbyte_cdk/destinations/vector_db_based/utils.py +35 -0
  28. airbyte_cdk/destinations/vector_db_based/writer.py +104 -0
  29. airbyte_cdk/entrypoint.py +414 -0
  30. airbyte_cdk/exception_handler.py +56 -0
  31. airbyte_cdk/logger.py +109 -0
  32. airbyte_cdk/models/__init__.py +72 -0
  33. airbyte_cdk/models/airbyte_protocol.py +88 -0
  34. airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
  35. airbyte_cdk/models/well_known_types.py +5 -0
  36. airbyte_cdk/py.typed +0 -0
  37. airbyte_cdk/sources/__init__.py +26 -0
  38. airbyte_cdk/sources/abstract_source.py +326 -0
  39. airbyte_cdk/sources/concurrent_source/__init__.py +8 -0
  40. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +255 -0
  41. airbyte_cdk/sources/concurrent_source/concurrent_source.py +165 -0
  42. airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +147 -0
  43. airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py +24 -0
  44. airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
  45. airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +115 -0
  46. airbyte_cdk/sources/config.py +27 -0
  47. airbyte_cdk/sources/connector_state_manager.py +161 -0
  48. airbyte_cdk/sources/declarative/__init__.py +3 -0
  49. airbyte_cdk/sources/declarative/async_job/__init__.py +0 -0
  50. airbyte_cdk/sources/declarative/async_job/job.py +52 -0
  51. airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +525 -0
  52. airbyte_cdk/sources/declarative/async_job/job_tracker.py +79 -0
  53. airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
  54. airbyte_cdk/sources/declarative/async_job/status.py +24 -0
  55. airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
  56. airbyte_cdk/sources/declarative/auth/__init__.py +8 -0
  57. airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +42 -0
  58. airbyte_cdk/sources/declarative/auth/jwt.py +197 -0
  59. airbyte_cdk/sources/declarative/auth/oauth.py +293 -0
  60. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +45 -0
  61. airbyte_cdk/sources/declarative/auth/token.py +267 -0
  62. airbyte_cdk/sources/declarative/auth/token_provider.py +82 -0
  63. airbyte_cdk/sources/declarative/checks/__init__.py +24 -0
  64. airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +61 -0
  65. airbyte_cdk/sources/declarative/checks/check_stream.py +56 -0
  66. airbyte_cdk/sources/declarative/checks/connection_checker.py +35 -0
  67. airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
  68. airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
  69. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +526 -0
  70. airbyte_cdk/sources/declarative/datetime/__init__.py +3 -0
  71. airbyte_cdk/sources/declarative/datetime/datetime_parser.py +65 -0
  72. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +118 -0
  73. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +3975 -0
  74. airbyte_cdk/sources/declarative/declarative_source.py +36 -0
  75. airbyte_cdk/sources/declarative/declarative_stream.py +241 -0
  76. airbyte_cdk/sources/declarative/decoders/__init__.py +33 -0
  77. airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +218 -0
  78. airbyte_cdk/sources/declarative/decoders/decoder.py +32 -0
  79. airbyte_cdk/sources/declarative/decoders/decoder_parser.py +30 -0
  80. airbyte_cdk/sources/declarative/decoders/json_decoder.py +65 -0
  81. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
  82. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
  83. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
  84. airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py +56 -0
  85. airbyte_cdk/sources/declarative/exceptions.py +9 -0
  86. airbyte_cdk/sources/declarative/extractors/__init__.py +21 -0
  87. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +86 -0
  88. airbyte_cdk/sources/declarative/extractors/http_selector.py +37 -0
  89. airbyte_cdk/sources/declarative/extractors/record_extractor.py +27 -0
  90. airbyte_cdk/sources/declarative/extractors/record_filter.py +91 -0
  91. airbyte_cdk/sources/declarative/extractors/record_selector.py +170 -0
  92. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +176 -0
  93. airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
  94. airbyte_cdk/sources/declarative/incremental/__init__.py +37 -0
  95. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +497 -0
  96. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +459 -0
  97. airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
  98. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +357 -0
  99. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +380 -0
  100. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
  101. airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
  102. airbyte_cdk/sources/declarative/interpolation/__init__.py +9 -0
  103. airbyte_cdk/sources/declarative/interpolation/filters.py +139 -0
  104. airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +66 -0
  105. airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +56 -0
  106. airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +52 -0
  107. airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +79 -0
  108. airbyte_cdk/sources/declarative/interpolation/interpolation.py +34 -0
  109. airbyte_cdk/sources/declarative/interpolation/jinja.py +161 -0
  110. airbyte_cdk/sources/declarative/interpolation/macros.py +191 -0
  111. airbyte_cdk/sources/declarative/manifest_declarative_source.py +421 -0
  112. airbyte_cdk/sources/declarative/migrations/__init__.py +0 -0
  113. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
  114. airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
  115. airbyte_cdk/sources/declarative/models/__init__.py +2 -0
  116. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2503 -0
  117. airbyte_cdk/sources/declarative/parsers/__init__.py +3 -0
  118. airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +157 -0
  119. airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +21 -0
  120. airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +172 -0
  121. airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +213 -0
  122. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +3407 -0
  123. airbyte_cdk/sources/declarative/partition_routers/__init__.py +29 -0
  124. airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
  125. airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
  126. airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +121 -0
  127. airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
  128. airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +63 -0
  129. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +437 -0
  130. airbyte_cdk/sources/declarative/requesters/README.md +56 -0
  131. airbyte_cdk/sources/declarative/requesters/__init__.py +9 -0
  132. airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +25 -0
  133. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +23 -0
  134. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +45 -0
  135. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +45 -0
  136. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +41 -0
  137. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +70 -0
  138. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +77 -0
  139. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +17 -0
  140. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +101 -0
  141. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +147 -0
  142. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
  143. airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +17 -0
  144. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +179 -0
  145. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +350 -0
  146. airbyte_cdk/sources/declarative/requesters/http_requester.py +433 -0
  147. airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +21 -0
  148. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +327 -0
  149. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +76 -0
  150. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +65 -0
  151. airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +25 -0
  152. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +98 -0
  153. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +102 -0
  154. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +71 -0
  155. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +48 -0
  156. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +66 -0
  157. airbyte_cdk/sources/declarative/requesters/request_option.py +117 -0
  158. airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +23 -0
  159. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +92 -0
  160. airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
  161. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +59 -0
  162. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +68 -0
  163. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +119 -0
  164. airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +79 -0
  165. airbyte_cdk/sources/declarative/requesters/request_path.py +15 -0
  166. airbyte_cdk/sources/declarative/requesters/requester.py +144 -0
  167. airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
  168. airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
  169. airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
  170. airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
  171. airbyte_cdk/sources/declarative/retrievers/__init__.py +19 -0
  172. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +124 -0
  173. airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
  174. airbyte_cdk/sources/declarative/retrievers/retriever.py +54 -0
  175. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +702 -0
  176. airbyte_cdk/sources/declarative/schema/__init__.py +25 -0
  177. airbyte_cdk/sources/declarative/schema/default_schema_loader.py +47 -0
  178. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +285 -0
  179. airbyte_cdk/sources/declarative/schema/inline_schema_loader.py +19 -0
  180. airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +92 -0
  181. airbyte_cdk/sources/declarative/schema/schema_loader.py +17 -0
  182. airbyte_cdk/sources/declarative/spec/__init__.py +7 -0
  183. airbyte_cdk/sources/declarative/spec/spec.py +48 -0
  184. airbyte_cdk/sources/declarative/stream_slicers/__init__.py +7 -0
  185. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +93 -0
  186. airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +25 -0
  187. airbyte_cdk/sources/declarative/transformations/__init__.py +17 -0
  188. airbyte_cdk/sources/declarative/transformations/add_fields.py +146 -0
  189. airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py +61 -0
  190. airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
  191. airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
  192. airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
  193. airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
  194. airbyte_cdk/sources/declarative/transformations/remove_fields.py +75 -0
  195. airbyte_cdk/sources/declarative/transformations/transformation.py +37 -0
  196. airbyte_cdk/sources/declarative/types.py +25 -0
  197. airbyte_cdk/sources/declarative/yaml_declarative_source.py +67 -0
  198. airbyte_cdk/sources/file_based/README.md +152 -0
  199. airbyte_cdk/sources/file_based/__init__.py +24 -0
  200. airbyte_cdk/sources/file_based/availability_strategy/__init__.py +11 -0
  201. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +73 -0
  202. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +149 -0
  203. airbyte_cdk/sources/file_based/config/__init__.py +0 -0
  204. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +153 -0
  205. airbyte_cdk/sources/file_based/config/avro_format.py +25 -0
  206. airbyte_cdk/sources/file_based/config/csv_format.py +210 -0
  207. airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
  208. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +99 -0
  209. airbyte_cdk/sources/file_based/config/jsonl_format.py +18 -0
  210. airbyte_cdk/sources/file_based/config/parquet_format.py +25 -0
  211. airbyte_cdk/sources/file_based/config/unstructured_format.py +102 -0
  212. airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py +81 -0
  213. airbyte_cdk/sources/file_based/discovery_policy/__init__.py +8 -0
  214. airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +21 -0
  215. airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +33 -0
  216. airbyte_cdk/sources/file_based/exceptions.py +159 -0
  217. airbyte_cdk/sources/file_based/file_based_source.py +466 -0
  218. airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py +123 -0
  219. airbyte_cdk/sources/file_based/file_based_stream_reader.py +209 -0
  220. airbyte_cdk/sources/file_based/file_record_data.py +22 -0
  221. airbyte_cdk/sources/file_based/file_types/__init__.py +37 -0
  222. airbyte_cdk/sources/file_based/file_types/avro_parser.py +233 -0
  223. airbyte_cdk/sources/file_based/file_types/csv_parser.py +527 -0
  224. airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
  225. airbyte_cdk/sources/file_based/file_types/file_transfer.py +30 -0
  226. airbyte_cdk/sources/file_based/file_types/file_type_parser.py +86 -0
  227. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +145 -0
  228. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +275 -0
  229. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +480 -0
  230. airbyte_cdk/sources/file_based/remote_file.py +18 -0
  231. airbyte_cdk/sources/file_based/schema_helpers.py +281 -0
  232. airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +17 -0
  233. airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +20 -0
  234. airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +52 -0
  235. airbyte_cdk/sources/file_based/stream/__init__.py +13 -0
  236. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +197 -0
  237. airbyte_cdk/sources/file_based/stream/concurrent/__init__.py +0 -0
  238. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +343 -0
  239. airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +9 -0
  240. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +59 -0
  241. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +313 -0
  242. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +83 -0
  243. airbyte_cdk/sources/file_based/stream/cursor/__init__.py +4 -0
  244. airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +66 -0
  245. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +149 -0
  246. airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +396 -0
  247. airbyte_cdk/sources/file_based/stream/identities_stream.py +49 -0
  248. airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +92 -0
  249. airbyte_cdk/sources/file_based/types.py +10 -0
  250. airbyte_cdk/sources/http_config.py +10 -0
  251. airbyte_cdk/sources/http_logger.py +55 -0
  252. airbyte_cdk/sources/message/__init__.py +19 -0
  253. airbyte_cdk/sources/message/repository.py +137 -0
  254. airbyte_cdk/sources/source.py +95 -0
  255. airbyte_cdk/sources/specs/transfer_modes.py +26 -0
  256. airbyte_cdk/sources/streams/__init__.py +8 -0
  257. airbyte_cdk/sources/streams/availability_strategy.py +84 -0
  258. airbyte_cdk/sources/streams/call_rate.py +704 -0
  259. airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
  260. airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
  261. airbyte_cdk/sources/streams/checkpoint/cursor.py +77 -0
  262. airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
  263. airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
  264. airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
  265. airbyte_cdk/sources/streams/concurrent/README.md +7 -0
  266. airbyte_cdk/sources/streams/concurrent/__init__.py +3 -0
  267. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +96 -0
  268. airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py +37 -0
  269. airbyte_cdk/sources/streams/concurrent/adapters.py +397 -0
  270. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +94 -0
  271. airbyte_cdk/sources/streams/concurrent/clamping.py +99 -0
  272. airbyte_cdk/sources/streams/concurrent/cursor.py +481 -0
  273. airbyte_cdk/sources/streams/concurrent/cursor_types.py +32 -0
  274. airbyte_cdk/sources/streams/concurrent/default_stream.py +102 -0
  275. airbyte_cdk/sources/streams/concurrent/exceptions.py +18 -0
  276. airbyte_cdk/sources/streams/concurrent/helpers.py +42 -0
  277. airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +64 -0
  278. airbyte_cdk/sources/streams/concurrent/partition_reader.py +45 -0
  279. airbyte_cdk/sources/streams/concurrent/partitions/__init__.py +3 -0
  280. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +48 -0
  281. airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py +18 -0
  282. airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
  283. airbyte_cdk/sources/streams/concurrent/partitions/types.py +38 -0
  284. airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py +0 -0
  285. airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +182 -0
  286. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +223 -0
  287. airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
  288. airbyte_cdk/sources/streams/core.py +703 -0
  289. airbyte_cdk/sources/streams/http/__init__.py +10 -0
  290. airbyte_cdk/sources/streams/http/availability_strategy.py +54 -0
  291. airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
  292. airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
  293. airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
  294. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
  295. airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
  296. airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
  297. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
  298. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
  299. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
  300. airbyte_cdk/sources/streams/http/exceptions.py +61 -0
  301. airbyte_cdk/sources/streams/http/http.py +673 -0
  302. airbyte_cdk/sources/streams/http/http_client.py +531 -0
  303. airbyte_cdk/sources/streams/http/rate_limiting.py +158 -0
  304. airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py +14 -0
  305. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +479 -0
  306. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +34 -0
  307. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +436 -0
  308. airbyte_cdk/sources/streams/http/requests_native_auth/token.py +83 -0
  309. airbyte_cdk/sources/streams/permissions/identities_stream.py +75 -0
  310. airbyte_cdk/sources/streams/utils/__init__.py +3 -0
  311. airbyte_cdk/sources/types.py +169 -0
  312. airbyte_cdk/sources/utils/__init__.py +7 -0
  313. airbyte_cdk/sources/utils/casing.py +12 -0
  314. airbyte_cdk/sources/utils/files_directory.py +15 -0
  315. airbyte_cdk/sources/utils/record_helper.py +53 -0
  316. airbyte_cdk/sources/utils/schema_helpers.py +230 -0
  317. airbyte_cdk/sources/utils/slice_logger.py +57 -0
  318. airbyte_cdk/sources/utils/transform.py +277 -0
  319. airbyte_cdk/sources/utils/types.py +7 -0
  320. airbyte_cdk/sql/__init__.py +0 -0
  321. airbyte_cdk/sql/_util/__init__.py +0 -0
  322. airbyte_cdk/sql/_util/hashing.py +34 -0
  323. airbyte_cdk/sql/_util/name_normalizers.py +92 -0
  324. airbyte_cdk/sql/constants.py +32 -0
  325. airbyte_cdk/sql/exceptions.py +235 -0
  326. airbyte_cdk/sql/secrets.py +123 -0
  327. airbyte_cdk/sql/shared/__init__.py +15 -0
  328. airbyte_cdk/sql/shared/catalog_providers.py +145 -0
  329. airbyte_cdk/sql/shared/sql_processor.py +786 -0
  330. airbyte_cdk/sql/types.py +160 -0
  331. airbyte_cdk/test/__init__.py +7 -0
  332. airbyte_cdk/test/catalog_builder.py +81 -0
  333. airbyte_cdk/test/entrypoint_wrapper.py +250 -0
  334. airbyte_cdk/test/mock_http/__init__.py +6 -0
  335. airbyte_cdk/test/mock_http/matcher.py +41 -0
  336. airbyte_cdk/test/mock_http/mocker.py +185 -0
  337. airbyte_cdk/test/mock_http/request.py +103 -0
  338. airbyte_cdk/test/mock_http/response.py +28 -0
  339. airbyte_cdk/test/mock_http/response_builder.py +237 -0
  340. airbyte_cdk/test/state_builder.py +33 -0
  341. airbyte_cdk/test/utils/__init__.py +1 -0
  342. airbyte_cdk/test/utils/data.py +24 -0
  343. airbyte_cdk/test/utils/http_mocking.py +16 -0
  344. airbyte_cdk/test/utils/manifest_only_fixtures.py +59 -0
  345. airbyte_cdk/test/utils/reading.py +26 -0
  346. airbyte_cdk/utils/__init__.py +10 -0
  347. airbyte_cdk/utils/airbyte_secrets_utils.py +80 -0
  348. airbyte_cdk/utils/analytics_message.py +25 -0
  349. airbyte_cdk/utils/constants.py +5 -0
  350. airbyte_cdk/utils/datetime_format_inferrer.py +94 -0
  351. airbyte_cdk/utils/datetime_helpers.py +499 -0
  352. airbyte_cdk/utils/event_timing.py +85 -0
  353. airbyte_cdk/utils/is_cloud_environment.py +18 -0
  354. airbyte_cdk/utils/mapping_helpers.py +162 -0
  355. airbyte_cdk/utils/message_utils.py +26 -0
  356. airbyte_cdk/utils/oneof_option_config.py +33 -0
  357. airbyte_cdk/utils/print_buffer.py +75 -0
  358. airbyte_cdk/utils/schema_inferrer.py +270 -0
  359. airbyte_cdk/utils/slice_hasher.py +37 -0
  360. airbyte_cdk/utils/spec_schema_transformations.py +26 -0
  361. airbyte_cdk/utils/stream_status_utils.py +43 -0
  362. airbyte_cdk/utils/traced_exception.py +145 -0
  363. airbyte_cdk-0.0.0.dev0.dist-info/LICENSE.txt +19 -0
  364. airbyte_cdk-0.0.0.dev0.dist-info/LICENSE_SHORT +1 -0
  365. airbyte_cdk-0.0.0.dev0.dist-info/METADATA +111 -0
  366. airbyte_cdk-0.0.0.dev0.dist-info/RECORD +368 -0
  367. airbyte_cdk-0.0.0.dev0.dist-info/WHEEL +4 -0
  368. airbyte_cdk-0.0.0.dev0.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,3975 @@
1
+ "$schema": http://json-schema.org/draft-07/schema#
2
+ "$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml
3
+ title: DeclarativeSource
4
+ type: object
5
+ description: An API source that extracts data according to its declarative components.
6
+ version: 1.0.0
7
+ required:
8
+ - type
9
+ - check
10
+ - version
11
+ anyOf:
12
+ - required:
13
+ - streams
14
+ - required:
15
+ - dynamic_streams
16
+ properties:
17
+ type:
18
+ type: string
19
+ enum: [DeclarativeSource]
20
+ check:
21
+ anyOf:
22
+ - "$ref": "#/definitions/CheckStream"
23
+ - "$ref": "#/definitions/CheckDynamicStream"
24
+ streams:
25
+ type: array
26
+ items:
27
+ anyOf:
28
+ - "$ref": "#/definitions/DeclarativeStream"
29
+ - "$ref": "#/definitions/StateDelegatingStream"
30
+ dynamic_streams:
31
+ type: array
32
+ items:
33
+ "$ref": "#/definitions/DynamicDeclarativeStream"
34
+ version:
35
+ type: string
36
+ description: The version of the Airbyte CDK used to build and test the source.
37
+ schemas:
38
+ "$ref": "#/definitions/Schemas"
39
+ definitions:
40
+ type: object
41
+ spec:
42
+ "$ref": "#/definitions/Spec"
43
+ concurrency_level:
44
+ "$ref": "#/definitions/ConcurrencyLevel"
45
+ api_budget:
46
+ "$ref": "#/definitions/HTTPAPIBudget"
47
+ max_concurrent_async_job_count:
48
+ title: Maximum Concurrent Asynchronous Jobs
49
+ description: Maximum number of concurrent asynchronous jobs to run. This property is only relevant for sources/streams that support asynchronous job execution through the AsyncRetriever (e.g. a report-based stream that initiates a job, polls the job status, and then fetches the job results). This is often set by the API's maximum number of concurrent jobs on the account level. Refer to the API's documentation for this information.
50
+ type: integer
51
+ metadata:
52
+ type: object
53
+ description: For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.
54
+ additionalProperties: true
55
+ description:
56
+ type: string
57
+ description: A description of the connector. It will be presented on the Source documentation page.
58
+ additionalProperties: false
59
+ definitions:
60
+ AddedFieldDefinition:
61
+ title: Definition Of Field To Add
62
+ description: Defines the field to add on a record.
63
+ type: object
64
+ required:
65
+ - type
66
+ - path
67
+ - value
68
+ properties:
69
+ type:
70
+ type: string
71
+ enum: [AddedFieldDefinition]
72
+ path:
73
+ title: Path
74
+ description: List of strings defining the path where to add the value on the record.
75
+ type: array
76
+ items:
77
+ type: string
78
+ examples:
79
+ - ["segment_id"]
80
+ - ["metadata", "segment_id"]
81
+ value:
82
+ title: Value
83
+ description: Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.
84
+ type: string
85
+ interpolation_context:
86
+ - config
87
+ - record
88
+ - stream_interval
89
+ - stream_partition
90
+ - stream_slice
91
+ examples:
92
+ - "{{ record['updates'] }}"
93
+ - "{{ record['MetaData']['LastUpdatedTime'] }}"
94
+ - "{{ stream_partition['segment_id'] }}"
95
+ value_type:
96
+ title: Value Type
97
+ description: Type of the value. If not specified, the type will be inferred from the value.
98
+ "$ref": "#/definitions/ValueType"
99
+ $parameters:
100
+ type: object
101
+ additionalProperties: true
102
+ AddFields:
103
+ title: Add Fields
104
+ description: Transformation which adds field to an output record. The path of the added field can be nested.
105
+ type: object
106
+ required:
107
+ - type
108
+ - fields
109
+ properties:
110
+ type:
111
+ type: string
112
+ enum: [AddFields]
113
+ fields:
114
+ title: Fields
115
+ description: List of transformations (path and corresponding value) that will be added to the record.
116
+ type: array
117
+ items:
118
+ "$ref": "#/definitions/AddedFieldDefinition"
119
+ condition:
120
+ description: Fields will be added if expression is evaluated to True.
121
+ type: string
122
+ default: ""
123
+ interpolation_context:
124
+ - config
125
+ - property
126
+ - parameters
127
+ examples:
128
+ - "{{ property|string == '' }}"
129
+ - "{{ property is integer }}"
130
+ - "{{ property|length > 5 }}"
131
+ - "{{ property == 'some_string_to_match' }}"
132
+ $parameters:
133
+ type: object
134
+ additionalProperties: true
135
+ ApiKeyAuthenticator:
136
+ title: API Key Authenticator
137
+ description: Authenticator for requests authenticated with an API token injected as an HTTP request header.
138
+ type: object
139
+ required:
140
+ - type
141
+ properties:
142
+ type:
143
+ type: string
144
+ enum: [ApiKeyAuthenticator]
145
+ api_token:
146
+ title: API Key
147
+ description: The API key to inject in the request. Fill it in the user inputs.
148
+ type: string
149
+ interpolation_context:
150
+ - config
151
+ examples:
152
+ - "{{ config['api_key'] }}"
153
+ - "Token token={{ config['api_key'] }}"
154
+ header:
155
+ title: Header Name
156
+ description: The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.
157
+ type: string
158
+ interpolation_context:
159
+ - config
160
+ examples:
161
+ - Authorization
162
+ - Api-Token
163
+ - X-Auth-Token
164
+ inject_into:
165
+ title: Inject API Key Into Outgoing HTTP Request
166
+ description: Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.
167
+ "$ref": "#/definitions/RequestOption"
168
+ examples:
169
+ - inject_into: header
170
+ field_name: Authorization
171
+ - inject_into: request_parameter
172
+ field_name: authKey
173
+ $parameters:
174
+ type: object
175
+ additionalProperties: true
176
+ AuthFlow:
177
+ title: "Auth flow"
178
+ description: |-
179
+ Additional and optional specification object to describe what an 'advanced' Auth flow would need to function.
180
+ - A connector should be able to fully function with the configuration as described by the ConnectorSpecification in a 'basic' mode.
181
+ - The 'advanced' mode provides easier UX for the user with UI improvements and automations. However, this requires further setup on the
182
+ server side by instance or workspace admins beforehand. The trade-off is that the user does not have to provide as many technical
183
+ inputs anymore and the auth process is faster and easier to complete.
184
+ type: object
185
+ properties:
186
+ auth_flow_type:
187
+ title: "Auth flow type"
188
+ description: "The type of auth to use"
189
+ type: string
190
+ enum: ["oauth2.0", "oauth1.0"] # Future auth types should be added here
191
+ predicate_key:
192
+ title: "Predicate key"
193
+ description: JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.
194
+ type: array
195
+ items:
196
+ type: string
197
+ examples:
198
+ - ["credentials", "auth_type"]
199
+ predicate_value:
200
+ title: "Predicate value"
201
+ description: Value of the predicate_key fields for the advanced auth to be applicable.
202
+ type: string
203
+ examples:
204
+ - "Oauth"
205
+ oauth_config_specification:
206
+ "$ref": "#/definitions/OAuthConfigSpecification"
207
+ BasicHttpAuthenticator:
208
+ title: Basic HTTP Authenticator
209
+ description: Authenticator for requests authenticated with the Basic HTTP authentication scheme, which encodes a username and an optional password in the Authorization request header.
210
+ type: object
211
+ required:
212
+ - type
213
+ - username
214
+ properties:
215
+ type:
216
+ type: string
217
+ enum: [BasicHttpAuthenticator]
218
+ username:
219
+ title: Username
220
+ description: The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.
221
+ type: string
222
+ interpolation_context:
223
+ - config
224
+ examples:
225
+ - "{{ config['username'] }}"
226
+ - "{{ config['api_key'] }}"
227
+ password:
228
+ title: Password
229
+ description: The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.
230
+ type: string
231
+ default: ""
232
+ interpolation_context:
233
+ - config
234
+ examples:
235
+ - "{{ config['password'] }}"
236
+ - ""
237
+ $parameters:
238
+ type: object
239
+ additionalProperties: true
240
+ BearerAuthenticator:
241
+ title: Bearer Token Authenticator
242
+ description: "Authenticator for requests authenticated with a bearer token injected as a request header of the form `Authorization: Bearer <token>`."
243
+ type: object
244
+ required:
245
+ - type
246
+ - api_token
247
+ properties:
248
+ type:
249
+ type: string
250
+ enum: [BearerAuthenticator]
251
+ api_token:
252
+ title: Bearer Token
253
+ description: Token to inject as request header for authenticating with the API.
254
+ type: string
255
+ interpolation_context:
256
+ - config
257
+ examples:
258
+ - "{{ config['api_key'] }}"
259
+ - "{{ config['token'] }}"
260
+ $parameters:
261
+ type: object
262
+ additionalProperties: true
263
+ SelectiveAuthenticator:
264
+ title: Selective Authenticator
265
+ description: Authenticator that selects concrete authenticator based on config property.
266
+ type: object
267
+ additionalProperties: true
268
+ required:
269
+ - type
270
+ - authenticators
271
+ - authenticator_selection_path
272
+ properties:
273
+ type:
274
+ type: string
275
+ enum: [SelectiveAuthenticator]
276
+ authenticator_selection_path:
277
+ title: Authenticator Selection Path
278
+ description: Path of the field in config with selected authenticator name
279
+ type: array
280
+ items:
281
+ type: string
282
+ examples:
283
+ - ["auth"]
284
+ - ["auth", "type"]
285
+ authenticators:
286
+ title: Authenticators
287
+ description: Authenticators to select from.
288
+ type: object
289
+ additionalProperties:
290
+ anyOf:
291
+ - "$ref": "#/definitions/ApiKeyAuthenticator"
292
+ - "$ref": "#/definitions/BasicHttpAuthenticator"
293
+ - "$ref": "#/definitions/BearerAuthenticator"
294
+ - "$ref": "#/definitions/CustomAuthenticator"
295
+ - "$ref": "#/definitions/OAuthAuthenticator"
296
+ - "$ref": "#/definitions/JwtAuthenticator"
297
+ - "$ref": "#/definitions/NoAuth"
298
+ - "$ref": "#/definitions/SessionTokenAuthenticator"
299
+ - "$ref": "#/definitions/LegacySessionTokenAuthenticator"
300
+ examples:
301
+ - authenticators:
302
+ token: "#/definitions/ApiKeyAuthenticator"
303
+ oauth: "#/definitions/OAuthAuthenticator"
304
+ jwt: "#/definitions/JwtAuthenticator"
305
+ $parameters:
306
+ type: object
307
+ additionalProperties: true
308
+ CheckStream:
309
+ title: Streams to Check
310
+ description: Defines the streams to try reading when running a check operation.
311
+ type: object
312
+ required:
313
+ - type
314
+ - stream_names
315
+ properties:
316
+ type:
317
+ type: string
318
+ enum: [CheckStream]
319
+ stream_names:
320
+ title: Stream Names
321
+ description: Names of the streams to try reading from when running a check operation.
322
+ type: array
323
+ items:
324
+ type: string
325
+ examples:
326
+ - ["users"]
327
+ - ["users", "contacts"]
328
+ CheckDynamicStream:
329
+ title: Dynamic Streams to Check
330
+ description: (This component is experimental. Use at your own risk.) Defines the dynamic streams to try reading when running a check operation.
331
+ type: object
332
+ required:
333
+ - type
334
+ - stream_count
335
+ properties:
336
+ type:
337
+ type: string
338
+ enum: [CheckDynamicStream]
339
+ stream_count:
340
+ title: Stream Count
341
+ description: Numbers of the streams to try reading from when running a check operation.
342
+ type: integer
343
+ use_check_availability:
344
+ title: Use Check Availability
345
+ description: Enables stream check availability. This field is automatically set by the CDK.
346
+ type: boolean
347
+ default: true
348
+ CompositeErrorHandler:
349
+ title: Composite Error Handler
350
+ description: Error handler that sequentially iterates over a list of error handlers.
351
+ type: object
352
+ required:
353
+ - type
354
+ - error_handlers
355
+ properties:
356
+ type:
357
+ type: string
358
+ enum: [CompositeErrorHandler]
359
+ error_handlers:
360
+ title: Error Handlers
361
+ description: List of error handlers to iterate on to determine how to handle a failed response.
362
+ type: array
363
+ items:
364
+ anyOf:
365
+ - "$ref": "#/definitions/CompositeErrorHandler"
366
+ - "$ref": "#/definitions/DefaultErrorHandler"
367
+ $parameters:
368
+ type: object
369
+ additionalProperties: true
370
+ ConcurrencyLevel:
371
+ title: Concurrency Level
372
+ description: Defines the amount of parallelization for the streams that are being synced. The factor of parallelization is how many partitions or streams are synced at the same time. For example, with a concurrency_level of 10, ten streams or partitions of data will processed at the same time. Note that a value of 1 could create deadlock if a stream has a very high number of partitions.
373
+ type: object
374
+ required:
375
+ - default_concurrency
376
+ properties:
377
+ type:
378
+ type: string
379
+ enum: [ConcurrencyLevel]
380
+ default_concurrency:
381
+ title: Default Concurrency
382
+ description: The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.
383
+ anyOf:
384
+ - type: integer
385
+ - type: string
386
+ interpolation_context:
387
+ - config
388
+ examples:
389
+ - 10
390
+ - "{{ config['num_workers'] or 10 }}"
391
+ max_concurrency:
392
+ title: Max Concurrency
393
+ description: The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.
394
+ type: integer
395
+ examples:
396
+ - 20
397
+ - 100
398
+ $parameters:
399
+ type: object
400
+ additionalProperties: true
401
+ ConstantBackoffStrategy:
402
+ title: Constant Backoff
403
+ description: Backoff strategy with a constant backoff interval.
404
+ type: object
405
+ required:
406
+ - type
407
+ - backoff_time_in_seconds
408
+ properties:
409
+ type:
410
+ type: string
411
+ enum: [ConstantBackoffStrategy]
412
+ backoff_time_in_seconds:
413
+ title: Backoff Time
414
+ description: Backoff time in seconds.
415
+ anyOf:
416
+ - type: number
417
+ - type: string
418
+ interpolation_context:
419
+ - config
420
+ examples:
421
+ - 30
422
+ - 30.5
423
+ - "{{ config['backoff_time'] }}"
424
+ $parameters:
425
+ type: object
426
+ additionalProperties: true
427
+ CursorPagination:
428
+ title: Cursor Pagination
429
+ description: Pagination strategy that evaluates an interpolated string to define the next page to fetch.
430
+ type: object
431
+ required:
432
+ - type
433
+ - cursor_value
434
+ properties:
435
+ type:
436
+ type: string
437
+ enum: [CursorPagination]
438
+ cursor_value:
439
+ title: Cursor Value
440
+ description: Value of the cursor defining the next page to fetch.
441
+ type: string
442
+ interpolation_context:
443
+ - config
444
+ - headers
445
+ - last_page_size
446
+ - last_record
447
+ - response
448
+ examples:
449
+ - "{{ headers.link.next.cursor }}"
450
+ - "{{ last_record['key'] }}"
451
+ - "{{ response['nextPage'] }}"
452
+ page_size:
453
+ title: Page Size
454
+ description: The number of records to include in each pages.
455
+ type: integer
456
+ examples:
457
+ - 100
458
+ stop_condition:
459
+ title: Stop Condition
460
+ description: Template string evaluating when to stop paginating.
461
+ type: string
462
+ interpolation_context:
463
+ - config
464
+ - headers
465
+ - last_record
466
+ - response
467
+ examples:
468
+ - "{{ response.data.has_more is false }}"
469
+ - "{{ 'next' not in headers['link'] }}"
470
+ $parameters:
471
+ type: object
472
+ additionalProperties: true
473
+ CustomAuthenticator:
474
+ title: Custom Authenticator
475
+ description: Authenticator component whose behavior is derived from a custom code implementation of the connector.
476
+ type: object
477
+ additionalProperties: true
478
+ required:
479
+ - type
480
+ - class_name
481
+ properties:
482
+ type:
483
+ type: string
484
+ enum: [CustomAuthenticator]
485
+ class_name:
486
+ title: Class Name
487
+ description: Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.
488
+ type: string
489
+ additionalProperties: true
490
+ examples:
491
+ - "source_railz.components.ShortLivedTokenAuthenticator"
492
+ $parameters:
493
+ type: object
494
+ additionalProperties: true
495
+ CustomBackoffStrategy:
496
+ title: Custom Backoff Strategy
497
+ description: Backoff strategy component whose behavior is derived from a custom code implementation of the connector.
498
+ type: object
499
+ additionalProperties: true
500
+ required:
501
+ - type
502
+ - class_name
503
+ properties:
504
+ type:
505
+ type: string
506
+ enum: [CustomBackoffStrategy]
507
+ class_name:
508
+ title: Class Name
509
+ description: Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.
510
+ type: string
511
+ examples:
512
+ - "source_railz.components.MyCustomBackoffStrategy"
513
+ $parameters:
514
+ type: object
515
+ additionalProperties: true
516
+ CustomErrorHandler:
517
+ title: Custom Error Handler
518
+ description: Error handler component whose behavior is derived from a custom code implementation of the connector.
519
+ type: object
520
+ additionalProperties: true
521
+ required:
522
+ - type
523
+ - class_name
524
+ properties:
525
+ type:
526
+ type: string
527
+ enum: [CustomErrorHandler]
528
+ class_name:
529
+ title: Class Name
530
+ description: Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.
531
+ type: string
532
+ examples:
533
+ - "source_railz.components.MyCustomErrorHandler"
534
+ $parameters:
535
+ type: object
536
+ additionalProperties: true
537
+ CustomIncrementalSync:
538
+ title: Custom Incremental Sync
539
+ description: Incremental component whose behavior is derived from a custom code implementation of the connector.
540
+ type: object
541
+ additionalProperties: true
542
+ required:
543
+ - type
544
+ - class_name
545
+ - cursor_field
546
+ properties:
547
+ type:
548
+ type: string
549
+ enum: [CustomIncrementalSync]
550
+ class_name:
551
+ title: Class Name
552
+ description: Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.
553
+ type: string
554
+ additionalProperties: true
555
+ examples:
556
+ - "source_railz.components.MyCustomIncrementalSync"
557
+ cursor_field:
558
+ description: The location of the value on a record that will be used as a bookmark during sync.
559
+ type: string
560
+ $parameters:
561
+ type: object
562
+ additionalProperties: true
563
+ CustomPaginationStrategy:
564
+ title: Custom Pagination Strategy
565
+ description: Pagination strategy component whose behavior is derived from a custom code implementation of the connector.
566
+ type: object
567
+ additionalProperties: true
568
+ required:
569
+ - type
570
+ - class_name
571
+ properties:
572
+ type:
573
+ type: string
574
+ enum: [CustomPaginationStrategy]
575
+ class_name:
576
+ title: Class Name
577
+ description: Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.
578
+ type: string
579
+ examples:
580
+ - "source_railz.components.MyCustomPaginationStrategy"
581
+ $parameters:
582
+ type: object
583
+ additionalProperties: true
584
+ CustomRecordExtractor:
585
+ title: Custom Record Extractor
586
+ description: Record extractor component whose behavior is derived from a custom code implementation of the connector.
587
+ type: object
588
+ additionalProperties: true
589
+ required:
590
+ - type
591
+ - class_name
592
+ properties:
593
+ type:
594
+ type: string
595
+ enum: [CustomRecordExtractor]
596
+ class_name:
597
+ title: Class Name
598
+ description: Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.
599
+ type: string
600
+ examples:
601
+ - "source_railz.components.MyCustomRecordExtractor"
602
+ $parameters:
603
+ type: object
604
+ additionalProperties: true
605
+ CustomRecordFilter:
606
+ title: Custom Record Filter
607
+ description: Record filter component whose behavior is derived from a custom code implementation of the connector.
608
+ type: object
609
+ additionalProperties: true
610
+ required:
611
+ - type
612
+ - class_name
613
+ properties:
614
+ type:
615
+ type: string
616
+ enum: [CustomRecordFilter]
617
+ class_name:
618
+ title: Class Name
619
+ description: Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.
620
+ type: string
621
+ examples:
622
+ - "source_railz.components.MyCustomCustomRecordFilter"
623
+ $parameters:
624
+ type: object
625
+ additionalProperties: true
626
+ CustomRequester:
627
+ title: Custom Requester
628
+ description: Requester component whose behavior is derived from a custom code implementation of the connector.
629
+ type: object
630
+ additionalProperties: true
631
+ required:
632
+ - type
633
+ - class_name
634
+ properties:
635
+ type:
636
+ type: string
637
+ enum: [CustomRequester]
638
+ class_name:
639
+ title: Class Name
640
+ description: Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.
641
+ type: string
642
+ additionalProperties: true
643
+ examples:
644
+ - "source_railz.components.MyCustomRecordExtractor"
645
+ $parameters:
646
+ type: object
647
+ additionalProperties: true
648
+ CustomRetriever:
649
+ title: Custom Retriever
650
+ description: Retriever component whose behavior is derived from a custom code implementation of the connector.
651
+ type: object
652
+ additionalProperties: true
653
+ required:
654
+ - type
655
+ - class_name
656
+ properties:
657
+ type:
658
+ type: string
659
+ enum: [CustomRetriever]
660
+ class_name:
661
+ title: Class Name
662
+ description: Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.
663
+ type: string
664
+ additionalProperties: true
665
+ examples:
666
+ - "source_railz.components.MyCustomRetriever"
667
+ $parameters:
668
+ type: object
669
+ additionalProperties: true
670
+ CustomPartitionRouter:
671
+ title: Custom Partition Router
672
+ description: Partition router component whose behavior is derived from a custom code implementation of the connector.
673
+ type: object
674
+ additionalProperties: true
675
+ required:
676
+ - type
677
+ - class_name
678
+ properties:
679
+ type:
680
+ type: string
681
+ enum: [CustomPartitionRouter]
682
+ class_name:
683
+ title: Class Name
684
+ description: Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.
685
+ type: string
686
+ examples:
687
+ - "source_railz.components.MyCustomPartitionRouter"
688
+ $parameters:
689
+ type: object
690
+ additionalProperties: true
691
+ CustomSchemaLoader:
692
+ title: Custom Schema Loader
693
+ description: Schema Loader component whose behavior is derived from a custom code implementation of the connector.
694
+ type: object
695
+ additionalProperties: true
696
+ required:
697
+ - type
698
+ - class_name
699
+ properties:
700
+ type:
701
+ type: string
702
+ enum: [CustomSchemaLoader]
703
+ class_name:
704
+ title: Class Name
705
+ description: Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.
706
+ type: string
707
+ examples:
708
+ - "source_railz.components.MyCustomSchemaLoader"
709
+ $parameters:
710
+ type: object
711
+ additionalProperties: true
712
+ CustomSchemaNormalization:
713
+ title: Custom Schema Normalization
714
+ description: Schema normalization component whose behavior is derived from a custom code implementation of the connector.
715
+ type: object
716
+ additionalProperties: true
717
+ required:
718
+ - type
719
+ - class_name
720
+ properties:
721
+ type:
722
+ type: string
723
+ enum: [CustomSchemaNormalization]
724
+ class_name:
725
+ title: Class Name
726
+ description: Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.
727
+ type: string
728
+ additionalProperties: true
729
+ examples:
730
+ - "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer"
731
+ $parameters:
732
+ type: object
733
+ additionalProperties: true
734
+ CustomStateMigration:
735
+ title: Custom State Migration
736
+ description: Apply a custom transformation on the input state.
737
+ type: object
738
+ additionalProperties: true
739
+ required:
740
+ - type
741
+ - class_name
742
+ properties:
743
+ type:
744
+ type: string
745
+ enum: [CustomStateMigration]
746
+ class_name:
747
+ title: Class Name
748
+ description: Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.
749
+ type: string
750
+ examples:
751
+ - "source_railz.components.MyCustomStateMigration"
752
+ $parameters:
753
+ type: object
754
+ additionalProperties: true
755
+ CustomTransformation:
756
+ title: Custom Transformation
757
+ description: Transformation component whose behavior is derived from a custom code implementation of the connector.
758
+ type: object
759
+ additionalProperties: true
760
+ required:
761
+ - type
762
+ - class_name
763
+ properties:
764
+ type:
765
+ type: string
766
+ enum: [CustomTransformation]
767
+ class_name:
768
+ title: Class Name
769
+ description: Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.
770
+ type: string
771
+ examples:
772
+ - "source_railz.components.MyCustomTransformation"
773
+ $parameters:
774
+ type: object
775
+ additionalProperties: true
776
+ LegacyToPerPartitionStateMigration:
777
+ title: Legacy To Per-partition-state Migration
778
+ description:
779
+ 'Transforms the input state for per-partitioned streams from the legacy format to the low-code format.
780
+ The cursor field and partition ID fields are automatically extracted from the stream''s DatetimebasedCursor and SubstreamPartitionRouter.
781
+
782
+ Example input state:
783
+ {
784
+ "13506132": {
785
+ "last_changed": "2022-12-27T08:34:39+00:00"
786
+ }
787
+ Example output state:
788
+ {
789
+ "partition": {"id": "13506132"},
790
+ "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}
791
+ }
792
+ '
793
+ type: object
794
+ additionalProperties: true
795
+ properties:
796
+ type:
797
+ type: string
798
+ enum: [LegacyToPerPartitionStateMigration]
799
+ IncrementingCountCursor:
800
+ title: Incrementing Count Cursor
801
+ description: Cursor that allows for incremental sync according to a continuously increasing integer.
802
+ type: object
803
+ required:
804
+ - type
805
+ - cursor_field
806
+ properties:
807
+ type:
808
+ type: string
809
+ enum: [IncrementingCountCursor]
810
+ cursor_field:
811
+ title: Cursor Field
812
+ description: The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.
813
+ type: string
814
+ interpolation_context:
815
+ - config
816
+ examples:
817
+ - "created_at"
818
+ - "{{ config['record_cursor'] }}"
819
+ start_value:
820
+ title: Start Value
821
+ description: The value that determines the earliest record that should be synced.
822
+ anyOf:
823
+ - type: string
824
+ - type: integer
825
+ interpolation_context:
826
+ - config
827
+ examples:
828
+ - 0
829
+ - "{{ config['start_value'] }}"
830
+ start_value_option:
831
+ title: Inject Start Value Into Outgoing HTTP Request
832
+ description: Optionally configures how the start value will be sent in requests to the source API.
833
+ "$ref": "#/definitions/RequestOption"
834
+ $parameters:
835
+ type: object
836
+ additionalProperties: true
837
+ DatetimeBasedCursor:
838
+ title: Datetime Based Cursor
839
+ description: Cursor to provide incremental capabilities over datetime.
840
+ type: object
841
+ required:
842
+ - type
843
+ - cursor_field
844
+ - datetime_format
845
+ - start_datetime
846
+ properties:
847
+ type:
848
+ type: string
849
+ enum: [DatetimeBasedCursor]
850
+ clamping:
851
+ title: Date Range Clamping
852
+ description: This option is used to adjust the upper and lower boundaries of each datetime window to beginning and end of the provided target period (day, week, month)
853
+ type: object
854
+ required:
855
+ - target
856
+ properties:
857
+ target:
858
+ title: Target
859
+ description: The period of time that datetime windows will be clamped by
860
+ # This should ideally be an enum. However, we don't use an enum because we want to allow for connectors
861
+ # to support interpolation on the connector config to get the target which is an arbitrary string
862
+ type: string
863
+ interpolation_context:
864
+ - config
865
+ examples:
866
+ - "DAY"
867
+ - "WEEK"
868
+ - "MONTH"
869
+ - "{{ config['target'] }}"
870
+ target_details:
871
+ type: object
872
+ additionalProperties: true
873
+ cursor_field:
874
+ title: Cursor Field
875
+ description: The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.
876
+ type: string
877
+ interpolation_context:
878
+ - config
879
+ examples:
880
+ - "created_at"
881
+ - "{{ config['record_cursor'] }}"
882
+ datetime_format:
883
+ title: Outgoing Datetime Format
884
+ description: |
885
+ The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
886
+ * **%s**: Epoch unix timestamp - `1686218963`
887
+ * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`
888
+ * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`
889
+ * **%a**: Weekday (abbreviated) - `Sun`
890
+ * **%A**: Weekday (full) - `Sunday`
891
+ * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)
892
+ * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`
893
+ * **%b**: Month (abbreviated) - `Jan`
894
+ * **%B**: Month (full) - `January`
895
+ * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`
896
+ * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`
897
+ * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`
898
+ * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`
899
+ * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`
900
+ * **%p**: AM/PM indicator
901
+ * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`
902
+ * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`
903
+ * **%f**: Microsecond (zero-padded to 6 digits) - `000000`
904
+ * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`
905
+ * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`
906
+ * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`
907
+ * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`
908
+ * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`
909
+ * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`
910
+ * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`
911
+ * **%x**: Date standard format - `08/16/1988`
912
+ * **%X**: Time standard format - `21:30:00`
913
+ * **%%**: Literal '%' character
914
+
915
+ Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).
916
+ type: string
917
+ examples:
918
+ - "%Y-%m-%dT%H:%M:%S.%f%z"
919
+ - "%Y-%m-%d"
920
+ - "%s"
921
+ - "%ms"
922
+ - "%s_as_float"
923
+ start_datetime:
924
+ title: Start Datetime
925
+ description: The datetime that determines the earliest record that should be synced.
926
+ anyOf:
927
+ - type: string
928
+ - "$ref": "#/definitions/MinMaxDatetime"
929
+ interpolation_context:
930
+ - config
931
+ examples:
932
+ - "2020-01-1T00:00:00Z"
933
+ - "{{ config['start_time'] }}"
934
+ cursor_datetime_formats:
935
+ title: Cursor Datetime Formats
936
+ description: The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.
937
+ type: array
938
+ items:
939
+ type: string
940
+ examples:
941
+ - "%Y-%m-%dT%H:%M:%S.%f%z"
942
+ - "%Y-%m-%d"
943
+ - "%s"
944
+ cursor_granularity:
945
+ title: Cursor Granularity
946
+ description:
947
+ Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should
948
+ be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.
949
+ type: string
950
+ examples:
951
+ - "PT1S"
952
+ end_datetime:
953
+ title: End Datetime
954
+ description: The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.
955
+ anyOf:
956
+ - type: string
957
+ - "$ref": "#/definitions/MinMaxDatetime"
958
+ interpolation_context:
959
+ - config
960
+ examples:
961
+ - "2021-01-1T00:00:00Z"
962
+ - "{{ now_utc() }}"
963
+ - "{{ day_delta(-1) }}"
964
+ end_time_option:
965
+ title: Inject End Time Into Outgoing HTTP Request
966
+ description: Optionally configures how the end datetime will be sent in requests to the source API.
967
+ "$ref": "#/definitions/RequestOption"
968
+ is_data_feed:
969
+ title: Whether the target API is formatted as a data feed
970
+ description: A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.
971
+ type: boolean
972
+ is_client_side_incremental:
973
+ title: Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)
974
+ description: If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.
975
+ type: boolean
976
+ is_compare_strictly:
977
+ title: Whether to skip requests if the start time equals the end time
978
+ description: Set to True if the target API does not accept queries where the start time equal the end time.
979
+ type: boolean
980
+ default: False
981
+ global_substream_cursor:
982
+ title: Whether to store cursor as one value instead of per partition
983
+ description: This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).
984
+ type: boolean
985
+ default: false
986
+ lookback_window:
987
+ title: Lookback Window
988
+ description: Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.
989
+ type: string
990
+ interpolation_context:
991
+ - config
992
+ examples:
993
+ - "P1D"
994
+ - "P{{ config['lookback_days'] }}D"
995
+ partition_field_end:
996
+ title: Partition Field End
997
+ description: Name of the partition start time field.
998
+ type: string
999
+ examples:
1000
+ - "ending_time"
1001
+ partition_field_start:
1002
+ title: Partition Field Start
1003
+ description: Name of the partition end time field.
1004
+ type: string
1005
+ examples:
1006
+ - "starting_time"
1007
+ start_time_option:
1008
+ title: Inject Start Time Into Outgoing HTTP Request
1009
+ description: Optionally configures how the start datetime will be sent in requests to the source API.
1010
+ "$ref": "#/definitions/RequestOption"
1011
+ step:
1012
+ title: Step
1013
+ description: The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.
1014
+ type: string
1015
+ examples:
1016
+ - "P1W"
1017
+ - "{{ config['step_increment'] }}"
1018
+ $parameters:
1019
+ type: object
1020
+ additionalProperties: true
1021
+ JwtAuthenticator:
1022
+ title: JWT Authenticator
1023
+ description: Authenticator for requests using JWT authentication flow.
1024
+ type: object
1025
+ required:
1026
+ - type
1027
+ - secret_key
1028
+ - algorithm
1029
+ properties:
1030
+ type:
1031
+ type: string
1032
+ enum: [JwtAuthenticator]
1033
+ secret_key:
1034
+ type: string
1035
+ description: Secret used to sign the JSON web token.
1036
+ examples:
1037
+ - "{{ config['secret_key'] }}"
1038
+ base64_encode_secret_key:
1039
+ type: boolean
1040
+ description: When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.
1041
+ default: False
1042
+ algorithm:
1043
+ type: string
1044
+ description: Algorithm used to sign the JSON web token.
1045
+ enum:
1046
+ [
1047
+ "HS256",
1048
+ "HS384",
1049
+ "HS512",
1050
+ "ES256",
1051
+ "ES256K",
1052
+ "ES384",
1053
+ "ES512",
1054
+ "RS256",
1055
+ "RS384",
1056
+ "RS512",
1057
+ "PS256",
1058
+ "PS384",
1059
+ "PS512",
1060
+ "EdDSA",
1061
+ ]
1062
+ examples:
1063
+ - ES256
1064
+ - HS256
1065
+ - RS256
1066
+ - "{{ config['algorithm'] }}"
1067
+ token_duration:
1068
+ type: integer
1069
+ title: Token Duration
1070
+ description: The amount of time in seconds a JWT token can be valid after being issued.
1071
+ default: 1200
1072
+ examples:
1073
+ - 1200
1074
+ - 3600
1075
+ header_prefix:
1076
+ type: string
1077
+ title: Header Prefix
1078
+ description: The prefix to be used within the Authentication header.
1079
+ examples:
1080
+ - "Bearer"
1081
+ - "Basic"
1082
+ jwt_headers:
1083
+ type: object
1084
+ title: JWT Headers
1085
+ description: JWT headers used when signing JSON web token.
1086
+ additionalProperties: false
1087
+ properties:
1088
+ kid:
1089
+ type: string
1090
+ title: Key Identifier
1091
+ description: Private key ID for user account.
1092
+ examples:
1093
+ - "{{ config['kid'] }}"
1094
+ typ:
1095
+ type: string
1096
+ title: Type
1097
+ description: The media type of the complete JWT.
1098
+ default: JWT
1099
+ examples:
1100
+ - JWT
1101
+ cty:
1102
+ type: string
1103
+ title: Content Type
1104
+ description: Content type of JWT header.
1105
+ examples:
1106
+ - JWT
1107
+ additional_jwt_headers:
1108
+ type: object
1109
+ title: Additional JWT Headers
1110
+ description: Additional headers to be included with the JWT headers object.
1111
+ additionalProperties: true
1112
+ jwt_payload:
1113
+ type: object
1114
+ title: JWT Payload
1115
+ description: JWT Payload used when signing JSON web token.
1116
+ additionalProperties: false
1117
+ properties:
1118
+ iss:
1119
+ type: string
1120
+ title: Issuer
1121
+ description: The user/principal that issued the JWT. Commonly a value unique to the user.
1122
+ examples:
1123
+ - "{{ config['iss'] }}"
1124
+ sub:
1125
+ type: string
1126
+ title: Subject
1127
+ description: The subject of the JWT. Commonly defined by the API.
1128
+ aud:
1129
+ type: string
1130
+ title: Audience
1131
+ description: The recipient that the JWT is intended for. Commonly defined by the API.
1132
+ examples:
1133
+ - "appstoreconnect-v1"
1134
+ additional_jwt_payload:
1135
+ type: object
1136
+ title: Additional JWT Payload Properties
1137
+ description: Additional properties to be added to the JWT payload.
1138
+ additionalProperties: true
1139
+ $parameters:
1140
+ type: object
1141
+ additionalProperties: true
1142
+ OAuthAuthenticator:
1143
+ title: OAuth2
1144
+ description: Authenticator for requests using OAuth 2.0 authorization flow.
1145
+ type: object
1146
+ required:
1147
+ - type
1148
+ properties:
1149
+ type:
1150
+ type: string
1151
+ enum: [OAuthAuthenticator]
1152
+ client_id_name:
1153
+ title: Client ID Property Name
1154
+ description: The name of the property to use to refresh the `access_token`.
1155
+ type: string
1156
+ default: "client_id"
1157
+ examples:
1158
+ - custom_app_id
1159
+ client_id:
1160
+ title: Client ID
1161
+ description: The OAuth client ID. Fill it in the user inputs.
1162
+ type: string
1163
+ examples:
1164
+ - "{{ config['client_id }}"
1165
+ - "{{ config['credentials']['client_id }}"
1166
+ client_secret_name:
1167
+ title: Client Secret Property Name
1168
+ description: The name of the property to use to refresh the `access_token`.
1169
+ type: string
1170
+ default: "client_secret"
1171
+ examples:
1172
+ - custom_app_secret
1173
+ client_secret:
1174
+ title: Client Secret
1175
+ description: The OAuth client secret. Fill it in the user inputs.
1176
+ type: string
1177
+ examples:
1178
+ - "{{ config['client_secret }}"
1179
+ - "{{ config['credentials']['client_secret }}"
1180
+ refresh_token_name:
1181
+ title: Refresh Token Property Name
1182
+ description: The name of the property to use to refresh the `access_token`.
1183
+ type: string
1184
+ default: "refresh_token"
1185
+ examples:
1186
+ - custom_app_refresh_value
1187
+ refresh_token:
1188
+ title: Refresh Token
1189
+ description: Credential artifact used to get a new access token.
1190
+ type: string
1191
+ examples:
1192
+ - "{{ config['refresh_token'] }}"
1193
+ - "{{ config['credentials]['refresh_token'] }}"
1194
+ token_refresh_endpoint:
1195
+ title: Token Refresh Endpoint
1196
+ description: The full URL to call to obtain a new access token.
1197
+ type: string
1198
+ examples:
1199
+ - https://connect.squareup.com/oauth2/token
1200
+ access_token_name:
1201
+ title: Access Token Property Name
1202
+ description: The name of the property which contains the access token in the response from the token refresh endpoint.
1203
+ type: string
1204
+ default: "access_token"
1205
+ examples:
1206
+ - access_token
1207
+ access_token_value:
1208
+ title: Access Token Value
1209
+ description: The value of the access_token to bypass the token refreshing using `refresh_token`.
1210
+ type: string
1211
+ examples:
1212
+ - secret_access_token_value
1213
+ expires_in_name:
1214
+ title: Token Expiry Property Name
1215
+ description: The name of the property which contains the expiry date in the response from the token refresh endpoint.
1216
+ type: string
1217
+ default: "expires_in"
1218
+ examples:
1219
+ - expires_in
1220
+ grant_type_name:
1221
+ title: Grant Type Property Name
1222
+ description: The name of the property to use to refresh the `access_token`.
1223
+ type: string
1224
+ default: "grant_type"
1225
+ examples:
1226
+ - custom_grant_type
1227
+ grant_type:
1228
+ title: Grant Type
1229
+ description: Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.
1230
+ type: string
1231
+ default: "refresh_token"
1232
+ examples:
1233
+ - refresh_token
1234
+ - client_credentials
1235
+ refresh_request_body:
1236
+ title: Refresh Request Body
1237
+ description: Body of the request sent to get a new access token.
1238
+ type: object
1239
+ additionalProperties: true
1240
+ examples:
1241
+ - applicationId: "{{ config['application_id'] }}"
1242
+ applicationSecret: "{{ config['application_secret'] }}"
1243
+ token: "{{ config['token'] }}"
1244
+ refresh_request_headers:
1245
+ title: Refresh Request Headers
1246
+ description: Headers of the request sent to get a new access token.
1247
+ type: object
1248
+ additionalProperties: true
1249
+ examples:
1250
+ - Authorization: "<AUTH_TOKEN>"
1251
+ Content-Type: "application/x-www-form-urlencoded"
1252
+ scopes:
1253
+ title: Scopes
1254
+ description: List of scopes that should be granted to the access token.
1255
+ type: array
1256
+ items:
1257
+ type: string
1258
+ examples:
1259
+ - [
1260
+ "crm.list.read",
1261
+ "crm.objects.contacts.read",
1262
+ "crm.schema.contacts.read",
1263
+ ]
1264
+ token_expiry_date:
1265
+ title: Token Expiry Date
1266
+ description: The access token expiry date.
1267
+ type: string
1268
+ examples:
1269
+ - 2023-04-06T07:12:10.421833+00:00
1270
+ - 1680842386
1271
+ token_expiry_date_format:
1272
+ title: Token Expiry Date Format
1273
+ description: The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.
1274
+ type: string
1275
+ examples:
1276
+ - "%Y-%m-%d %H:%M:%S.%f+00:00"
1277
+ refresh_token_updater:
1278
+ title: Token Updater
1279
+ description: When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.
1280
+ properties:
1281
+ refresh_token_name:
1282
+ title: Refresh Token Property Name
1283
+ description: The name of the property which contains the updated refresh token in the response from the token refresh endpoint.
1284
+ type: string
1285
+ default: "refresh_token"
1286
+ examples:
1287
+ - "refresh_token"
1288
+ access_token_config_path:
1289
+ title: Config Path To Access Token
1290
+ description: Config path to the access token. Make sure the field actually exists in the config.
1291
+ type: array
1292
+ items:
1293
+ type: string
1294
+ default: ["credentials", "access_token"]
1295
+ examples:
1296
+ - ["credentials", "access_token"]
1297
+ - ["access_token"]
1298
+ refresh_token_config_path:
1299
+ title: Config Path To Refresh Token
1300
+ description: Config path to the access token. Make sure the field actually exists in the config.
1301
+ type: array
1302
+ items:
1303
+ type: string
1304
+ default: ["credentials", "refresh_token"]
1305
+ examples:
1306
+ - ["credentials", "refresh_token"]
1307
+ - ["refresh_token"]
1308
+ token_expiry_date_config_path:
1309
+ title: Config Path To Expiry Date
1310
+ description: Config path to the expiry date. Make sure actually exists in the config.
1311
+ type: array
1312
+ items:
1313
+ type: string
1314
+ default: ["credentials", "token_expiry_date"]
1315
+ examples:
1316
+ - ["credentials", "token_expiry_date"]
1317
+ refresh_token_error_status_codes:
1318
+ title: Refresh Token Error Status Codes
1319
+ description: Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error
1320
+ type: array
1321
+ items:
1322
+ type: integer
1323
+ default: []
1324
+ examples:
1325
+ - [400, 500]
1326
+ refresh_token_error_key:
1327
+ title: Refresh Token Error Key
1328
+ description: Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).
1329
+ type: string
1330
+ default: ""
1331
+ examples:
1332
+ - "error"
1333
+ refresh_token_error_values:
1334
+ title: Refresh Token Error Values
1335
+ description: 'List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error'
1336
+ type: array
1337
+ items:
1338
+ type: string
1339
+ default: []
1340
+ examples:
1341
+ - ["invalid_grant", "invalid_permissions"]
1342
+ profile_assertion:
1343
+ title: Profile Assertion
1344
+ description: The authenticator being used to authenticate the client authenticator.
1345
+ "$ref": "#/definitions/JwtAuthenticator"
1346
+ use_profile_assertion:
1347
+ title: Use Profile Assertion
1348
+ description: Enable using profile assertion as a flow for OAuth authorization.
1349
+ type: boolean
1350
+ default: false
1351
+ $parameters:
1352
+ type: object
1353
+ additionalProperties: true
1354
+ DeclarativeStream:
1355
+ title: Declarative Stream
1356
+ description: A stream whose behavior is described by a set of declarative low code components.
1357
+ type: object
1358
+ additionalProperties: true
1359
+ required:
1360
+ - type
1361
+ - retriever
1362
+ properties:
1363
+ type:
1364
+ type: string
1365
+ enum: [DeclarativeStream]
1366
+ retriever:
1367
+ title: Retriever
1368
+ description: Component used to coordinate how records are extracted across stream slices and request pages.
1369
+ anyOf:
1370
+ - "$ref": "#/definitions/AsyncRetriever"
1371
+ - "$ref": "#/definitions/CustomRetriever"
1372
+ - "$ref": "#/definitions/SimpleRetriever"
1373
+ incremental_sync:
1374
+ title: Incremental Sync
1375
+ description: Component used to fetch data incrementally based on a time field in the data.
1376
+ anyOf:
1377
+ - "$ref": "#/definitions/CustomIncrementalSync"
1378
+ - "$ref": "#/definitions/DatetimeBasedCursor"
1379
+ - "$ref": "#/definitions/IncrementingCountCursor"
1380
+ name:
1381
+ title: Name
1382
+ description: The stream name.
1383
+ type: string
1384
+ default: ""
1385
+ example:
1386
+ - "Users"
1387
+ primary_key:
1388
+ title: Primary Key
1389
+ description: The primary key of the stream.
1390
+ "$ref": "#/definitions/PrimaryKey"
1391
+ default: ""
1392
+ schema_loader:
1393
+ title: Schema Loader
1394
+ description: Component used to retrieve the schema for the current stream.
1395
+ anyOf:
1396
+ - "$ref": "#/definitions/DynamicSchemaLoader"
1397
+ - "$ref": "#/definitions/InlineSchemaLoader"
1398
+ - "$ref": "#/definitions/JsonFileSchemaLoader"
1399
+ - "$ref": "#/definitions/CustomSchemaLoader"
1400
+ # TODO we have move the transformation to the RecordSelector level in the code but kept this here for
1401
+ # compatibility reason. We should eventually move this to align with the code.
1402
+ transformations:
1403
+ title: Transformations
1404
+ description: A list of transformations to be applied to each output record.
1405
+ type: array
1406
+ items:
1407
+ anyOf:
1408
+ - "$ref": "#/definitions/AddFields"
1409
+ - "$ref": "#/definitions/CustomTransformation"
1410
+ - "$ref": "#/definitions/RemoveFields"
1411
+ - "$ref": "#/definitions/KeysToLower"
1412
+ - "$ref": "#/definitions/KeysToSnakeCase"
1413
+ - "$ref": "#/definitions/FlattenFields"
1414
+ - "$ref": "#/definitions/DpathFlattenFields"
1415
+ - "$ref": "#/definitions/KeysReplace"
1416
+ state_migrations:
1417
+ title: State Migrations
1418
+ description: Array of state migrations to be applied on the input state
1419
+ type: array
1420
+ items:
1421
+ anyOf:
1422
+ - "$ref": "#/definitions/LegacyToPerPartitionStateMigration"
1423
+ - "$ref": "#/definitions/CustomStateMigration"
1424
+ default: []
1425
+ file_uploader:
1426
+ title: File Uploader
1427
+ description: (experimental) Describes how to fetch a file
1428
+ type: object
1429
+ required:
1430
+ - type
1431
+ - requester
1432
+ - download_target_extractor
1433
+ properties:
1434
+ type:
1435
+ type: string
1436
+ enum: [ FileUploader ]
1437
+ requester:
1438
+ description: Requester component that describes how to prepare HTTP requests to send to the source API.
1439
+ anyOf:
1440
+ - "$ref": "#/definitions/CustomRequester"
1441
+ - "$ref": "#/definitions/HttpRequester"
1442
+ download_target_extractor:
1443
+ description: Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response
1444
+ anyOf:
1445
+ - "$ref": "#/definitions/CustomRecordExtractor"
1446
+ - "$ref": "#/definitions/DpathExtractor"
1447
+ file_extractor:
1448
+ description: Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content
1449
+ anyOf:
1450
+ - "$ref": "#/definitions/CustomRecordExtractor"
1451
+ - "$ref": "#/definitions/DpathExtractor"
1452
+ filename_extractor:
1453
+ description: Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.
1454
+ type: string
1455
+ interpolation_context:
1456
+ - config
1457
+ - record
1458
+ examples:
1459
+ - "{{ record.id }}/{{ record.file_name }}/"
1460
+ - "{{ record.id }}_{{ record.file_name }}/"
1461
+ $parameters:
1462
+ type: object
1463
+ additional_properties: true
1464
+ HTTPAPIBudget:
1465
+ title: HTTP API Budget
1466
+ description: >
1467
+ Defines how many requests can be made to the API in a given time frame. `HTTPAPIBudget` extracts the remaining
1468
+ call count and the reset time from HTTP response headers using the header names provided by
1469
+ `ratelimit_remaining_header` and `ratelimit_reset_header`. Only requests using `HttpRequester`
1470
+ are rate-limited; custom components that bypass `HttpRequester` are not covered by this budget.
1471
+ type: object
1472
+ required:
1473
+ - type
1474
+ - policies
1475
+ properties:
1476
+ type:
1477
+ type: string
1478
+ enum: [HTTPAPIBudget]
1479
+ policies:
1480
+ title: Policies
1481
+ description: List of call rate policies that define how many calls are allowed.
1482
+ type: array
1483
+ items:
1484
+ anyOf:
1485
+ - "$ref": "#/definitions/FixedWindowCallRatePolicy"
1486
+ - "$ref": "#/definitions/MovingWindowCallRatePolicy"
1487
+ - "$ref": "#/definitions/UnlimitedCallRatePolicy"
1488
+ ratelimit_reset_header:
1489
+ title: Rate Limit Reset Header
1490
+ description: The HTTP response header name that indicates when the rate limit resets.
1491
+ type: string
1492
+ default: "ratelimit-reset"
1493
+ ratelimit_remaining_header:
1494
+ title: Rate Limit Remaining Header
1495
+ description: The HTTP response header name that indicates the number of remaining allowed calls.
1496
+ type: string
1497
+ default: "ratelimit-remaining"
1498
+ status_codes_for_ratelimit_hit:
1499
+ title: Status Codes for Rate Limit Hit
1500
+ description: List of HTTP status codes that indicate a rate limit has been hit.
1501
+ type: array
1502
+ items:
1503
+ type: integer
1504
+ default: [429]
1505
+ additionalProperties: true
1506
+ FixedWindowCallRatePolicy:
1507
+ title: Fixed Window Call Rate Policy
1508
+ description: A policy that allows a fixed number of calls within a specific time window.
1509
+ type: object
1510
+ required:
1511
+ - type
1512
+ - period
1513
+ - call_limit
1514
+ - matchers
1515
+ properties:
1516
+ type:
1517
+ type: string
1518
+ enum: [FixedWindowCallRatePolicy]
1519
+ period:
1520
+ title: Period
1521
+ description: The time interval for the rate limit window.
1522
+ type: string
1523
+ call_limit:
1524
+ title: Call Limit
1525
+ description: The maximum number of calls allowed within the period.
1526
+ type: integer
1527
+ matchers:
1528
+ title: Matchers
1529
+ description: List of matchers that define which requests this policy applies to.
1530
+ type: array
1531
+ items:
1532
+ "$ref": "#/definitions/HttpRequestRegexMatcher"
1533
+ additionalProperties: true
1534
+ MovingWindowCallRatePolicy:
1535
+ title: Moving Window Call Rate Policy
1536
+ description: A policy that allows a fixed number of calls within a moving time window.
1537
+ type: object
1538
+ required:
1539
+ - type
1540
+ - rates
1541
+ - matchers
1542
+ properties:
1543
+ type:
1544
+ type: string
1545
+ enum: [MovingWindowCallRatePolicy]
1546
+ rates:
1547
+ title: Rates
1548
+ description: List of rates that define the call limits for different time intervals.
1549
+ type: array
1550
+ items:
1551
+ "$ref": "#/definitions/Rate"
1552
+ matchers:
1553
+ title: Matchers
1554
+ description: List of matchers that define which requests this policy applies to.
1555
+ type: array
1556
+ items:
1557
+ "$ref": "#/definitions/HttpRequestRegexMatcher"
1558
+ additionalProperties: true
1559
+ UnlimitedCallRatePolicy:
1560
+ title: Unlimited Call Rate Policy
1561
+ description: A policy that allows unlimited calls for specific requests.
1562
+ type: object
1563
+ required:
1564
+ - type
1565
+ - matchers
1566
+ properties:
1567
+ type:
1568
+ type: string
1569
+ enum: [UnlimitedCallRatePolicy]
1570
+ matchers:
1571
+ title: Matchers
1572
+ description: List of matchers that define which requests this policy applies to.
1573
+ type: array
1574
+ items:
1575
+ "$ref": "#/definitions/HttpRequestRegexMatcher"
1576
+ additionalProperties: true
1577
+ Rate:
1578
+ title: Rate
1579
+ description: Defines a rate limit with a specific number of calls allowed within a time interval.
1580
+ type: object
1581
+ required:
1582
+ - limit
1583
+ - interval
1584
+ properties:
1585
+ limit:
1586
+ title: Limit
1587
+ description: The maximum number of calls allowed within the interval.
1588
+ anyOf:
1589
+ - type: integer
1590
+ - type: string
1591
+ interpolation_context:
1592
+ - config
1593
+ interval:
1594
+ title: Interval
1595
+ description: The time interval for the rate limit.
1596
+ type: string
1597
+ examples:
1598
+ - "PT1H"
1599
+ - "P1D"
1600
+ additionalProperties: true
1601
+ HttpRequestRegexMatcher:
1602
+ title: HTTP Request Matcher
1603
+ description: >
1604
+ Matches HTTP requests based on method, base URL, URL path pattern, query parameters, and headers.
1605
+ Use `url_base` to specify the scheme and host (without trailing slash) and
1606
+ `url_path_pattern` to apply a regex to the request path.
1607
+ type: object
1608
+ properties:
1609
+ method:
1610
+ title: Method
1611
+ description: The HTTP method to match (e.g., GET, POST).
1612
+ type: string
1613
+ url_base:
1614
+ title: URL Base
1615
+ description: The base URL (scheme and host, e.g. "https://api.example.com") to match.
1616
+ type: string
1617
+ url_path_pattern:
1618
+ title: URL Path Pattern
1619
+ description: A regular expression pattern to match the URL path.
1620
+ type: string
1621
+ params:
1622
+ title: Parameters
1623
+ description: The query parameters to match.
1624
+ type: object
1625
+ additionalProperties: true
1626
+ headers:
1627
+ title: Headers
1628
+ description: The headers to match.
1629
+ type: object
1630
+ additionalProperties: true
1631
+ additionalProperties: true
1632
+ DefaultErrorHandler:
1633
+ title: Default Error Handler
1634
+ description: Component defining how to handle errors. Default behavior includes only retrying server errors (HTTP 5XX) and too many requests (HTTP 429) with an exponential backoff.
1635
+ type: object
1636
+ required:
1637
+ - type
1638
+ properties:
1639
+ type:
1640
+ type: string
1641
+ enum: [DefaultErrorHandler]
1642
+ backoff_strategies:
1643
+ title: Backoff Strategies
1644
+ description: List of backoff strategies to use to determine how long to wait before retrying a retryable request.
1645
+ type: array
1646
+ items:
1647
+ anyOf:
1648
+ - "$ref": "#/definitions/ConstantBackoffStrategy"
1649
+ - "$ref": "#/definitions/CustomBackoffStrategy"
1650
+ - "$ref": "#/definitions/ExponentialBackoffStrategy"
1651
+ - "$ref": "#/definitions/WaitTimeFromHeader"
1652
+ - "$ref": "#/definitions/WaitUntilTimeFromHeader"
1653
+ max_retries:
1654
+ title: Max Retry Count
1655
+ description: The maximum number of time to retry a retryable request before giving up and failing.
1656
+ type: integer
1657
+ default: 5
1658
+ examples:
1659
+ - 5
1660
+ - 0
1661
+ - 10
1662
+ response_filters:
1663
+ title: Response Filters
1664
+ description: List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.
1665
+ type: array
1666
+ items:
1667
+ "$ref": "#/definitions/HttpResponseFilter"
1668
+ $parameters:
1669
+ type: object
1670
+ additional_properties: true
1671
+ DefaultPaginator:
1672
+ title: Default Paginator
1673
+ description: Default pagination implementation to request pages of results with a fixed size until the pagination strategy no longer returns a next_page_token.
1674
+ type: object
1675
+ required:
1676
+ - type
1677
+ - pagination_strategy
1678
+ properties:
1679
+ type:
1680
+ type: string
1681
+ enum: [DefaultPaginator]
1682
+ pagination_strategy:
1683
+ title: Pagination Strategy
1684
+ description: Strategy defining how records are paginated.
1685
+ anyOf:
1686
+ - "$ref": "#/definitions/CursorPagination"
1687
+ - "$ref": "#/definitions/CustomPaginationStrategy"
1688
+ - "$ref": "#/definitions/OffsetIncrement"
1689
+ - "$ref": "#/definitions/PageIncrement"
1690
+ page_size_option:
1691
+ "$ref": "#/definitions/RequestOption"
1692
+ page_token_option:
1693
+ anyOf:
1694
+ - "$ref": "#/definitions/RequestOption"
1695
+ - "$ref": "#/definitions/RequestPath"
1696
+ $parameters:
1697
+ type: object
1698
+ additionalProperties: true
1699
+ DpathExtractor:
1700
+ title: Dpath Extractor
1701
+ description: Record extractor that searches a decoded response over a path defined as an array of fields.
1702
+ type: object
1703
+ required:
1704
+ - type
1705
+ - field_path
1706
+ properties:
1707
+ type:
1708
+ type: string
1709
+ enum: [DpathExtractor]
1710
+ field_path:
1711
+ title: Field Path
1712
+ description: List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).
1713
+ type: array
1714
+ items:
1715
+ type: string
1716
+ interpolation_context:
1717
+ - config
1718
+ examples:
1719
+ - ["data"]
1720
+ - ["data", "records"]
1721
+ - ["data", "{{ parameters.name }}"]
1722
+ - ["data", "*", "record"]
1723
+ $parameters:
1724
+ type: object
1725
+ additionalProperties: true
1726
+ ResponseToFileExtractor:
1727
+ title: CSV To File Extractor
1728
+ description: A record extractor designed for handling large responses that may exceed memory limits (to prevent OOM issues). It downloads a CSV file to disk, reads the data from disk, and deletes the file once it has been fully processed.
1729
+ type: object
1730
+ required:
1731
+ - type
1732
+ properties:
1733
+ type:
1734
+ type: string
1735
+ enum: [ResponseToFileExtractor]
1736
+ $parameters:
1737
+ type: object
1738
+ additionalProperties: true
1739
+ ExponentialBackoffStrategy:
1740
+ title: Exponential Backoff
1741
+ description: Backoff strategy with an exponential backoff interval. The interval is defined as factor * 2^attempt_count.
1742
+ type: object
1743
+ required:
1744
+ - type
1745
+ properties:
1746
+ type:
1747
+ type: string
1748
+ enum: [ExponentialBackoffStrategy]
1749
+ factor:
1750
+ title: Factor
1751
+ description: Multiplicative constant applied on each retry.
1752
+ anyOf:
1753
+ - type: number
1754
+ - type: string
1755
+ default: 5
1756
+ interpolation_context:
1757
+ - config
1758
+ examples:
1759
+ - 5
1760
+ - 5.5
1761
+ - "10"
1762
+ $parameters:
1763
+ type: object
1764
+ additionalProperties: true
1765
+ SessionTokenAuthenticator:
1766
+ type: object
1767
+ required:
1768
+ - type
1769
+ - login_requester
1770
+ - session_token_path
1771
+ - request_authentication
1772
+ properties:
1773
+ type:
1774
+ type: string
1775
+ enum: [SessionTokenAuthenticator]
1776
+ login_requester:
1777
+ title: Login Requester
1778
+ description: Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.
1779
+ "$ref": "#/definitions/HttpRequester"
1780
+ examples:
1781
+ - type: HttpRequester
1782
+ url_base: "https://my_api.com"
1783
+ path: "/login"
1784
+ authenticator:
1785
+ type: BasicHttpAuthenticator
1786
+ username: "{{ config.username }}"
1787
+ password: "{{ config.password }}"
1788
+ session_token_path:
1789
+ title: Session Token Path
1790
+ description: The path in the response body returned from the login requester to the session token.
1791
+ examples:
1792
+ - ["access_token"]
1793
+ - ["result", "token"]
1794
+ type: array
1795
+ items:
1796
+ type: string
1797
+ expiration_duration:
1798
+ title: Expiration Duration
1799
+ description: The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.
1800
+ type: string
1801
+ examples:
1802
+ - "PT1H"
1803
+ - "P1D"
1804
+ request_authentication:
1805
+ title: Data Request Authentication
1806
+ description: Authentication method to use for requests sent to the API, specifying how to inject the session token.
1807
+ anyOf:
1808
+ - "$ref": "#/definitions/SessionTokenRequestApiKeyAuthenticator"
1809
+ - "$ref": "#/definitions/SessionTokenRequestBearerAuthenticator"
1810
+ decoder:
1811
+ title: Decoder
1812
+ description: Component used to decode the response.
1813
+ anyOf:
1814
+ - "$ref": "#/definitions/JsonDecoder"
1815
+ - "$ref": "#/definitions/XmlDecoder"
1816
+ $parameters:
1817
+ type: object
1818
+ additionalProperties: true
1819
+ SessionTokenRequestApiKeyAuthenticator:
1820
+ type: object
1821
+ title: API Key Authenticator
1822
+ description: Authenticator for requests using the session token as an API key that's injected into the request.
1823
+ required:
1824
+ - type
1825
+ - inject_into
1826
+ properties:
1827
+ type:
1828
+ enum: [ApiKey]
1829
+ inject_into:
1830
+ title: Inject API Key Into Outgoing HTTP Request
1831
+ description: Configure how the API Key will be sent in requests to the source API.
1832
+ "$ref": "#/definitions/RequestOption"
1833
+ examples:
1834
+ - inject_into: header
1835
+ field_name: Authorization
1836
+ - inject_into: request_parameter
1837
+ field_name: authKey
1838
+ SessionTokenRequestBearerAuthenticator:
1839
+ title: Bearer Authenticator
1840
+ description: Authenticator for requests using the session token as a standard bearer token.
1841
+ required:
1842
+ - type
1843
+ properties:
1844
+ type:
1845
+ enum: [Bearer]
1846
+ HttpRequester:
1847
+ title: HTTP Requester
1848
+ description: Requester submitting HTTP requests and extracting records from the response.
1849
+ type: object
1850
+ required:
1851
+ - type
1852
+ - url_base
1853
+ properties:
1854
+ type:
1855
+ type: string
1856
+ enum: [HttpRequester]
1857
+ url_base:
1858
+ title: API Base URL
1859
+ description: Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
1860
+ type: string
1861
+ interpolation_context:
1862
+ - config
1863
+ - next_page_token
1864
+ - stream_interval
1865
+ - stream_partition
1866
+ - stream_slice
1867
+ - creation_response
1868
+ - polling_response
1869
+ - download_target
1870
+ examples:
1871
+ - "https://connect.squareup.com/v2"
1872
+ - "{{ config['base_url'] or 'https://app.posthog.com'}}/api"
1873
+ - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups"
1874
+ - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}"
1875
+ path:
1876
+ title: URL Path
1877
+ description: Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
1878
+ type: string
1879
+ interpolation_context:
1880
+ - config
1881
+ - next_page_token
1882
+ - stream_interval
1883
+ - stream_partition
1884
+ - stream_slice
1885
+ - creation_response
1886
+ - polling_response
1887
+ - download_target
1888
+ examples:
1889
+ - "/products"
1890
+ - "/quotes/{{ stream_partition['id'] }}/quote_line_groups"
1891
+ - "/trades/{{ config['symbol_id'] }}/history"
1892
+ authenticator:
1893
+ title: Authenticator
1894
+ description: Authentication method to use for requests sent to the API.
1895
+ anyOf:
1896
+ - "$ref": "#/definitions/ApiKeyAuthenticator"
1897
+ - "$ref": "#/definitions/BasicHttpAuthenticator"
1898
+ - "$ref": "#/definitions/BearerAuthenticator"
1899
+ - "$ref": "#/definitions/CustomAuthenticator"
1900
+ - "$ref": "#/definitions/OAuthAuthenticator"
1901
+ - "$ref": "#/definitions/JwtAuthenticator"
1902
+ - "$ref": "#/definitions/NoAuth"
1903
+ - "$ref": "#/definitions/SessionTokenAuthenticator"
1904
+ - "$ref": "#/definitions/LegacySessionTokenAuthenticator"
1905
+ - "$ref": "#/definitions/SelectiveAuthenticator"
1906
+ error_handler:
1907
+ title: Error Handler
1908
+ description: Error handler component that defines how to handle errors.
1909
+ anyOf:
1910
+ - "$ref": "#/definitions/DefaultErrorHandler"
1911
+ - "$ref": "#/definitions/CustomErrorHandler"
1912
+ - "$ref": "#/definitions/CompositeErrorHandler"
1913
+ http_method:
1914
+ title: HTTP Method
1915
+ description: The HTTP method used to fetch data from the source (can be GET or POST).
1916
+ type: string
1917
+ enum:
1918
+ - GET
1919
+ - POST
1920
+ default: GET
1921
+ examples:
1922
+ - GET
1923
+ - POST
1924
+ request_body_data:
1925
+ title: Request Body Payload (Non-JSON)
1926
+ description: Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.
1927
+ anyOf:
1928
+ - type: string
1929
+ - type: object
1930
+ additionalProperties:
1931
+ type: string
1932
+ interpolation_context:
1933
+ - next_page_token
1934
+ - stream_interval
1935
+ - stream_partition
1936
+ - stream_slice
1937
+ examples:
1938
+ - |
1939
+ [{"clause": {"type": "timestamp", "operator": 10, "parameters":
1940
+ [{"value": {{ stream_interval['start_time'] | int * 1000 }} }]
1941
+ }, "orderBy": 1, "columnName": "Timestamp"}]/
1942
+ request_body_json:
1943
+ title: Request Body JSON Payload
1944
+ description: Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.
1945
+ anyOf:
1946
+ - type: string
1947
+ - type: object
1948
+ additionalProperties: true
1949
+ interpolation_context:
1950
+ - next_page_token
1951
+ - stream_interval
1952
+ - stream_partition
1953
+ - stream_slice
1954
+ examples:
1955
+ - sort_order: "ASC"
1956
+ sort_field: "CREATED_AT"
1957
+ - key: "{{ config['value'] }}"
1958
+ - sort:
1959
+ field: "updated_at"
1960
+ order: "ascending"
1961
+ request_headers:
1962
+ title: Request Headers
1963
+ description: Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.
1964
+ anyOf:
1965
+ - type: string
1966
+ - type: object
1967
+ additionalProperties:
1968
+ type: string
1969
+ interpolation_context:
1970
+ - next_page_token
1971
+ - stream_interval
1972
+ - stream_partition
1973
+ - stream_slice
1974
+ examples:
1975
+ - Output-Format: JSON
1976
+ - Version: "{{ config['version'] }}"
1977
+ request_parameters:
1978
+ title: Query Parameters
1979
+ description: Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.
1980
+ anyOf:
1981
+ - type: string
1982
+ - type: object
1983
+ additionalProperties:
1984
+ type: string
1985
+ interpolation_context:
1986
+ - next_page_token
1987
+ - stream_interval
1988
+ - stream_partition
1989
+ - stream_slice
1990
+ examples:
1991
+ - unit: "day"
1992
+ - query: 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
1993
+ - searchIn: "{{ ','.join(config.get('search_in', [])) }}"
1994
+ - sort_by[asc]: updated_at
1995
+ use_cache:
1996
+ title: Use Cache
1997
+ description: Enables stream requests caching. This field is automatically set by the CDK.
1998
+ type: boolean
1999
+ default: false
2000
+ $parameters:
2001
+ type: object
2002
+ additionalProperties: true
2003
+ HttpResponseFilter:
2004
+ description: A filter that is used to select on properties of the HTTP response received. When used with additional filters, a response will be selected if it matches any of the filter's criteria.
2005
+ type: object
2006
+ required:
2007
+ - type
2008
+ properties:
2009
+ type:
2010
+ type: string
2011
+ enum: [HttpResponseFilter]
2012
+ action:
2013
+ title: Action
2014
+ description: Action to execute if a response matches the filter.
2015
+ type: string
2016
+ enum:
2017
+ - SUCCESS
2018
+ - FAIL
2019
+ - RETRY
2020
+ - IGNORE
2021
+ - RATE_LIMITED
2022
+ examples:
2023
+ - SUCCESS
2024
+ - FAIL
2025
+ - RETRY
2026
+ - IGNORE
2027
+ - RATE_LIMITED
2028
+ failure_type:
2029
+ title: Failure Type
2030
+ description: Failure type of traced exception if a response matches the filter.
2031
+ type: string
2032
+ enum:
2033
+ - system_error
2034
+ - config_error
2035
+ - transient_error
2036
+ examples:
2037
+ - system_error
2038
+ - config_error
2039
+ - transient_error
2040
+ error_message:
2041
+ title: Error Message
2042
+ description: Error Message to display if the response matches the filter.
2043
+ type: string
2044
+ interpolation_context:
2045
+ - config
2046
+ - response
2047
+ - headers
2048
+ error_message_contains:
2049
+ title: Error Message Substring
2050
+ description: Match the response if its error message contains the substring.
2051
+ type: string
2052
+ example:
2053
+ - This API operation is not enabled for this site
2054
+ http_codes:
2055
+ title: HTTP Codes
2056
+ description: Match the response if its HTTP code is included in this list.
2057
+ type: array
2058
+ items:
2059
+ type: integer
2060
+ uniqueItems: true
2061
+ examples:
2062
+ - [420, 429]
2063
+ - [500]
2064
+ predicate:
2065
+ title: Predicate
2066
+ description: Match the response if the predicate evaluates to true.
2067
+ type: string
2068
+ interpolation_context:
2069
+ - response
2070
+ - headers
2071
+ examples:
2072
+ - "{{ 'Too much requests' in response }}"
2073
+ - "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}"
2074
+ $parameters:
2075
+ type: object
2076
+ additionalProperties: true
2077
+ ComplexFieldType:
2078
+ title: Schema Field Type
2079
+ description: (This component is experimental. Use at your own risk.) Represents a complex field type.
2080
+ type: object
2081
+ required:
2082
+ - field_type
2083
+ properties:
2084
+ field_type:
2085
+ type: string
2086
+ items:
2087
+ anyOf:
2088
+ - type: string
2089
+ - "$ref": "#/definitions/ComplexFieldType"
2090
+ TypesMap:
2091
+ title: Types Map
2092
+ description: (This component is experimental. Use at your own risk.) Represents a mapping between a current type and its corresponding target type.
2093
+ type: object
2094
+ required:
2095
+ - target_type
2096
+ - current_type
2097
+ properties:
2098
+ target_type:
2099
+ anyOf:
2100
+ - type: string
2101
+ - type: array
2102
+ items:
2103
+ type: string
2104
+ - "$ref": "#/definitions/ComplexFieldType"
2105
+ current_type:
2106
+ anyOf:
2107
+ - type: string
2108
+ - type: array
2109
+ items:
2110
+ type: string
2111
+ condition:
2112
+ type: string
2113
+ interpolation_context:
2114
+ - raw_schema
2115
+ SchemaTypeIdentifier:
2116
+ title: Schema Type Identifier
2117
+ description: (This component is experimental. Use at your own risk.) Identifies schema details for dynamic schema extraction and processing.
2118
+ type: object
2119
+ required:
2120
+ - key_pointer
2121
+ properties:
2122
+ type:
2123
+ type: string
2124
+ enum: [SchemaTypeIdentifier]
2125
+ schema_pointer:
2126
+ title: Schema Path
2127
+ description: List of nested fields defining the schema field path to extract. Defaults to [].
2128
+ type: array
2129
+ default: []
2130
+ items:
2131
+ type: string
2132
+ interpolation_context:
2133
+ - config
2134
+ key_pointer:
2135
+ title: Key Path
2136
+ description: List of potentially nested fields describing the full path of the field key to extract.
2137
+ type: array
2138
+ items:
2139
+ type: string
2140
+ interpolation_context:
2141
+ - config
2142
+ type_pointer:
2143
+ title: Type Path
2144
+ description: List of potentially nested fields describing the full path of the field type to extract.
2145
+ type: array
2146
+ items:
2147
+ type: string
2148
+ interpolation_context:
2149
+ - config
2150
+ types_mapping:
2151
+ type: array
2152
+ items:
2153
+ "$ref": "#/definitions/TypesMap"
2154
+ $parameters:
2155
+ type: object
2156
+ additionalProperties: true
2157
+ DynamicSchemaLoader:
2158
+ title: Dynamic Schema Loader
2159
+ description: (This component is experimental. Use at your own risk.) Loads a schema by extracting data from retrieved records.
2160
+ type: object
2161
+ required:
2162
+ - type
2163
+ - retriever
2164
+ - schema_type_identifier
2165
+ properties:
2166
+ type:
2167
+ type: string
2168
+ enum: [DynamicSchemaLoader]
2169
+ retriever:
2170
+ title: Retriever
2171
+ description: Component used to coordinate how records are extracted across stream slices and request pages.
2172
+ anyOf:
2173
+ - "$ref": "#/definitions/AsyncRetriever"
2174
+ - "$ref": "#/definitions/CustomRetriever"
2175
+ - "$ref": "#/definitions/SimpleRetriever"
2176
+ schema_transformations:
2177
+ title: Schema Transformations
2178
+ description: A list of transformations to be applied to the schema.
2179
+ type: array
2180
+ items:
2181
+ anyOf:
2182
+ - "$ref": "#/definitions/AddFields"
2183
+ - "$ref": "#/definitions/CustomTransformation"
2184
+ - "$ref": "#/definitions/RemoveFields"
2185
+ - "$ref": "#/definitions/KeysToLower"
2186
+ - "$ref": "#/definitions/KeysToSnakeCase"
2187
+ - "$ref": "#/definitions/FlattenFields"
2188
+ - "$ref": "#/definitions/DpathFlattenFields"
2189
+ - "$ref": "#/definitions/KeysReplace"
2190
+ schema_type_identifier:
2191
+ "$ref": "#/definitions/SchemaTypeIdentifier"
2192
+ $parameters:
2193
+ type: object
2194
+ additionalProperties: true
2195
+ InlineSchemaLoader:
2196
+ title: Inline Schema Loader
2197
+ description: Loads a schema that is defined directly in the manifest file.
2198
+ type: object
2199
+ required:
2200
+ - type
2201
+ properties:
2202
+ type:
2203
+ type: string
2204
+ enum: [InlineSchemaLoader]
2205
+ schema:
2206
+ title: Schema
2207
+ description: Describes a streams' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.
2208
+ type: object
2209
+ JsonFileSchemaLoader:
2210
+ title: Json File Schema Loader
2211
+ description: Loads the schema from a json file.
2212
+ type: object
2213
+ required:
2214
+ - type
2215
+ properties:
2216
+ type:
2217
+ type: string
2218
+ enum: [JsonFileSchemaLoader]
2219
+ file_path:
2220
+ title: File Path
2221
+ description: Path to the JSON file defining the schema. The path is relative to the connector module's root.
2222
+ type: string
2223
+ interpolation_context:
2224
+ - config
2225
+ example:
2226
+ - "./schemas/users.json"
2227
+ $parameters:
2228
+ type: object
2229
+ additionalProperties: true
2230
+ JsonDecoder:
2231
+ title: Json Decoder
2232
+ type: object
2233
+ required:
2234
+ - type
2235
+ properties:
2236
+ type:
2237
+ type: string
2238
+ enum: [JsonDecoder]
2239
+ JsonlDecoder:
2240
+ title: JSONL Decoder
2241
+ description: Use this if the response consists of JSON objects separated by new lines (`\n`) in JSONL format.
2242
+ type: object
2243
+ required:
2244
+ - type
2245
+ properties:
2246
+ type:
2247
+ type: string
2248
+ enum: [JsonlDecoder]
2249
+ KeysToLower:
2250
+ title: Keys to Lower Case
2251
+ description: A transformation that renames all keys to lower case.
2252
+ type: object
2253
+ required:
2254
+ - type
2255
+ properties:
2256
+ type:
2257
+ type: string
2258
+ enum: [KeysToLower]
2259
+ $parameters:
2260
+ type: object
2261
+ additionalProperties: true
2262
+ KeysToSnakeCase:
2263
+ title: Key to Snake Case
2264
+ description: A transformation that renames all keys to snake case.
2265
+ type: object
2266
+ required:
2267
+ - type
2268
+ properties:
2269
+ type:
2270
+ type: string
2271
+ enum: [KeysToSnakeCase]
2272
+ $parameters:
2273
+ type: object
2274
+ additionalProperties: true
2275
+ FlattenFields:
2276
+ title: Flatten Fields
2277
+ description: A transformation that flatten record to single level format.
2278
+ type: object
2279
+ required:
2280
+ - type
2281
+ properties:
2282
+ type:
2283
+ type: string
2284
+ enum: [FlattenFields]
2285
+ flatten_lists:
2286
+ title: Flatten Lists
2287
+ description: Whether to flatten lists or leave it as is. Default is True.
2288
+ type: boolean
2289
+ default: true
2290
+ $parameters:
2291
+ type: object
2292
+ additionalProperties: true
2293
+ DpathFlattenFields:
2294
+ title: Dpath Flatten Fields
2295
+ description: A transformation that flatten field values to the to top of the record.
2296
+ type: object
2297
+ required:
2298
+ - type
2299
+ - field_path
2300
+ properties:
2301
+ type:
2302
+ type: string
2303
+ enum: [DpathFlattenFields]
2304
+ field_path:
2305
+ title: Field Path
2306
+ description: A path to field that needs to be flattened.
2307
+ type: array
2308
+ items:
2309
+ type: string
2310
+ examples:
2311
+ - ["data"]
2312
+ - ["data", "*", "field"]
2313
+ delete_origin_value:
2314
+ title: Delete Origin Value
2315
+ description: Whether to delete the origin value or keep it. Default is False.
2316
+ type: boolean
2317
+ replace_record:
2318
+ title: Replace Origin Record
2319
+ description: Whether to replace the origin record or not. Default is False.
2320
+ type: boolean
2321
+ $parameters:
2322
+ type: object
2323
+ additionalProperties: true
2324
+ KeysReplace:
2325
+ title: Keys Replace
2326
+ description: A transformation that replaces symbols in keys.
2327
+ type: object
2328
+ required:
2329
+ - type
2330
+ - old
2331
+ - new
2332
+ properties:
2333
+ type:
2334
+ type: string
2335
+ enum: [KeysReplace]
2336
+ old:
2337
+ type: string
2338
+ title: Old value
2339
+ description: Old value to replace.
2340
+ examples:
2341
+ - " "
2342
+ - "{{ record.id }}"
2343
+ - "{{ config['id'] }}"
2344
+ - "{{ stream_slice['id'] }}"
2345
+ interpolation_context:
2346
+ - config
2347
+ - record
2348
+ - stream_slice
2349
+ new:
2350
+ type: string
2351
+ title: New value
2352
+ description: New value to set.
2353
+ examples:
2354
+ - "_"
2355
+ - "{{ record.id }}"
2356
+ - "{{ config['id'] }}"
2357
+ - "{{ stream_slice['id'] }}"
2358
+ interpolation_context:
2359
+ - config
2360
+ - record
2361
+ - stream_slice
2362
+ $parameters:
2363
+ type: object
2364
+ additionalProperties: true
2365
+ IterableDecoder:
2366
+ title: Iterable Decoder
2367
+ description: Use this if the response consists of strings separated by new lines (`\n`). The Decoder will wrap each row into a JSON object with the `record` key.
2368
+ type: object
2369
+ required:
2370
+ - type
2371
+ properties:
2372
+ type:
2373
+ type: string
2374
+ enum: [IterableDecoder]
2375
+ XmlDecoder:
2376
+ title: XML Decoder
2377
+ description: Use this if the response is XML.
2378
+ type: object
2379
+ required:
2380
+ - type
2381
+ properties:
2382
+ type:
2383
+ type: string
2384
+ enum: [XmlDecoder]
2385
+ CustomDecoder:
2386
+ title: Custom Decoder
2387
+ description: Use this to implement custom decoder logic.
2388
+ type: object
2389
+ additionalProperties: true
2390
+ required:
2391
+ - type
2392
+ - class_name
2393
+ properties:
2394
+ type:
2395
+ type: string
2396
+ enum: [CustomDecoder]
2397
+ class_name:
2398
+ title: Class Name
2399
+ description: Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.
2400
+ type: string
2401
+ additionalProperties: true
2402
+ examples:
2403
+ - "source_amazon_ads.components.GzipJsonlDecoder"
2404
+ $parameters:
2405
+ type: object
2406
+ additionalProperties: true
2407
+ ZipfileDecoder:
2408
+ title: Zipfile Decoder
2409
+ description: Decoder for response data that is returned as zipfile(s).
2410
+ type: object
2411
+ additionalProperties: true
2412
+ required:
2413
+ - type
2414
+ - decoder
2415
+ properties:
2416
+ type:
2417
+ type: string
2418
+ enum: [ZipfileDecoder]
2419
+ decoder:
2420
+ title: Parser
2421
+ description: Parser to parse the decompressed data from the zipfile(s).
2422
+ anyOf:
2423
+ - "$ref": "#/definitions/CsvDecoder"
2424
+ - "$ref": "#/definitions/GzipDecoder"
2425
+ - "$ref": "#/definitions/JsonDecoder"
2426
+ - "$ref": "#/definitions/JsonlDecoder"
2427
+ ListPartitionRouter:
2428
+ title: List Partition Router
2429
+ description: A Partition router that specifies a list of attributes where each attribute describes a portion of the complete data set for a stream. During a sync, each value is iterated over and can be used as input to outbound API requests.
2430
+ type: object
2431
+ required:
2432
+ - type
2433
+ - cursor_field
2434
+ - values
2435
+ properties:
2436
+ type:
2437
+ type: string
2438
+ enum: [ListPartitionRouter]
2439
+ cursor_field:
2440
+ title: Current Partition Value Identifier
2441
+ description: While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition['my_key'] }}" where "my_key" is the value of the cursor_field.
2442
+ type: string
2443
+ interpolation_context:
2444
+ - config
2445
+ examples:
2446
+ - "section"
2447
+ - "{{ config['section_key'] }}"
2448
+ values:
2449
+ title: Partition Values
2450
+ description: The list of attributes being iterated over and used as input for the requests made to the source API.
2451
+ anyOf:
2452
+ - type: string
2453
+ - type: array
2454
+ items:
2455
+ type: string
2456
+ interpolation_context:
2457
+ - config
2458
+ examples:
2459
+ - ["section_a", "section_b", "section_c"]
2460
+ - "{{ config['sections'] }}"
2461
+ request_option:
2462
+ title: Inject Partition Value Into Outgoing HTTP Request
2463
+ description: A request option describing where the list value should be injected into and under what field name if applicable.
2464
+ "$ref": "#/definitions/RequestOption"
2465
+ $parameters:
2466
+ type: object
2467
+ additionalProperties: true
2468
+ MinMaxDatetime:
2469
+ title: Min-Max Datetime
2470
+ description: Compares the provided date against optional minimum or maximum times. The max_datetime serves as the ceiling and will be returned when datetime exceeds it. The min_datetime serves as the floor.
2471
+ type: object
2472
+ required:
2473
+ - type
2474
+ - datetime
2475
+ properties:
2476
+ type:
2477
+ type: string
2478
+ enum: [MinMaxDatetime]
2479
+ datetime:
2480
+ title: Datetime
2481
+ description: Datetime value.
2482
+ type: string
2483
+ interpolation_context:
2484
+ - config
2485
+ examples:
2486
+ - 2021-01-01
2487
+ - 2021-01-01T00:00:00Z
2488
+ - "{{ config['start_time'] }}"
2489
+ datetime_format:
2490
+ title: Datetime Format
2491
+ description: |
2492
+ Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
2493
+ * **%s**: Epoch unix timestamp - `1686218963`
2494
+ * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`
2495
+ * **%ms**: Epoch unix timestamp - `1686218963123`
2496
+ * **%a**: Weekday (abbreviated) - `Sun`
2497
+ * **%A**: Weekday (full) - `Sunday`
2498
+ * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)
2499
+ * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`
2500
+ * **%b**: Month (abbreviated) - `Jan`
2501
+ * **%B**: Month (full) - `January`
2502
+ * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`
2503
+ * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`
2504
+ * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`
2505
+ * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`
2506
+ * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`
2507
+ * **%p**: AM/PM indicator
2508
+ * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`
2509
+ * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`
2510
+ * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`
2511
+ * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`
2512
+ * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`
2513
+ * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`
2514
+ * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`
2515
+ * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`
2516
+ * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`
2517
+ * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`
2518
+ * **%x**: Date representation - `08/16/1988`
2519
+ * **%X**: Time representation - `21:30:00`
2520
+ * **%%**: Literal '%' character
2521
+
2522
+ Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).
2523
+ type: string
2524
+ default: ""
2525
+ examples:
2526
+ - "%Y-%m-%dT%H:%M:%S.%f%z"
2527
+ - "%Y-%m-%d"
2528
+ - "%s"
2529
+ max_datetime:
2530
+ title: Max Datetime
2531
+ description: Ceiling applied on the datetime value. Must be formatted with the datetime_format field.
2532
+ type: string
2533
+ interpolation_context:
2534
+ - config
2535
+ examples:
2536
+ - "2021-01-01T00:00:00Z"
2537
+ - "2021-01-01"
2538
+ min_datetime:
2539
+ title: Min Datetime
2540
+ description: Floor applied on the datetime value. Must be formatted with the datetime_format field.
2541
+ type: string
2542
+ interpolation_context:
2543
+ - config
2544
+ examples:
2545
+ - "2010-01-01T00:00:00Z"
2546
+ - "2010-01-01"
2547
+ $parameters:
2548
+ type: object
2549
+ additionalProperties: true
2550
+ NoAuth:
2551
+ title: No Authentication
2552
+ description: Authenticator for requests requiring no authentication.
2553
+ type: object
2554
+ required:
2555
+ - type
2556
+ properties:
2557
+ type:
2558
+ type: string
2559
+ enum: [NoAuth]
2560
+ $parameters:
2561
+ type: object
2562
+ additionalProperties: true
2563
+ NoPagination:
2564
+ title: No Pagination
2565
+ description: Pagination implementation that never returns a next page.
2566
+ type: object
2567
+ required:
2568
+ - type
2569
+ properties:
2570
+ type:
2571
+ type: string
2572
+ enum: [NoPagination]
2573
+ OAuthConfigSpecification:
2574
+ title: OAuth Config Specification
2575
+ description: Specification describing how an 'advanced' Auth flow would need to function.
2576
+ type: object
2577
+ additionalProperties: true
2578
+ properties:
2579
+ oauth_user_input_from_connector_config_specification:
2580
+ title: "OAuth user input"
2581
+ description: |-
2582
+ OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.
2583
+ Must be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification
2584
+ using special annotation 'path_in_connector_config'.
2585
+ These are input values the user is entering through the UI to authenticate to the connector, that might also shared
2586
+ as inputs for syncing data via the connector.
2587
+ Examples:
2588
+ if no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]
2589
+ if connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,
2590
+ oauth_user_input_from_connector_config_specification={
2591
+ app_id: {
2592
+ type: string
2593
+ path_in_connector_config: ['app_id']
2594
+ }
2595
+ }
2596
+ if connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,
2597
+ oauth_user_input_from_connector_config_specification={
2598
+ app_id: {
2599
+ type: string
2600
+ path_in_connector_config: ['info', 'app_id']
2601
+ }
2602
+ }
2603
+ type: object
2604
+ examples:
2605
+ - app_id:
2606
+ type: string
2607
+ path_in_connector_config: ["app_id"]
2608
+ - app_id:
2609
+ type: string
2610
+ path_in_connector_config: ["info", "app_id"]
2611
+ oauth_connector_input_specification:
2612
+ title: DeclarativeOAuth Connector Specification
2613
+ description: |-
2614
+ The DeclarativeOAuth specific blob.
2615
+ Pertains to the fields defined by the connector relating to the OAuth flow.
2616
+
2617
+ Interpolation capabilities:
2618
+ - The variables placeholders are declared as `{{my_var}}`.
2619
+ - The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.
2620
+
2621
+ - The allowed interpolation context is:
2622
+ + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}
2623
+ + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}
2624
+ + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}
2625
+ + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}
2626
+ + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}
2627
+
2628
+ Examples:
2629
+ - The TikTok Marketing DeclarativeOAuth spec:
2630
+ {
2631
+ "oauth_connector_input_specification": {
2632
+ "type": "object",
2633
+ "additionalProperties": false,
2634
+ "properties": {
2635
+ "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",
2636
+ "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",
2637
+ "access_token_params": {
2638
+ "{{ auth_code_key }}": "{{ auth_code_value }}",
2639
+ "{{ client_id_key }}": "{{ client_id_value }}",
2640
+ "{{ client_secret_key }}": "{{ client_secret_value }}"
2641
+ },
2642
+ "access_token_headers": {
2643
+ "Content-Type": "application/json",
2644
+ "Accept": "application/json"
2645
+ },
2646
+ "extract_output": ["data.access_token"],
2647
+ "client_id_key": "app_id",
2648
+ "client_secret_key": "secret",
2649
+ "auth_code_key": "auth_code"
2650
+ }
2651
+ }
2652
+ }
2653
+ type: object
2654
+ additionalProperties: true
2655
+ required:
2656
+ - consent_url
2657
+ - access_token_url
2658
+ properties:
2659
+ consent_url:
2660
+ title: Consent URL
2661
+ type: string
2662
+ description: |-
2663
+ The DeclarativeOAuth Specific string URL string template to initiate the authentication.
2664
+ The placeholders are replaced during the processing to provide neccessary values.
2665
+ examples:
2666
+ - https://domain.host.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}
2667
+ - https://endpoint.host.com/oauth2/authorize?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}&{{scope_key}}={{{{scope_value}} | urlEncoder}}&{{state_key}}={{state_value}}&subdomain={{subdomain}}
2668
+ scope:
2669
+ title: Scopes
2670
+ type: string
2671
+ description: |-
2672
+ The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.
2673
+ examples:
2674
+ - user:read user:read_orders workspaces:read
2675
+ access_token_url:
2676
+ title: Access Token URL
2677
+ type: string
2678
+ description: |-
2679
+ The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.
2680
+ The placeholders are replaced during the processing to provide neccessary values.
2681
+ examples:
2682
+ - https://auth.host.com/oauth2/token?{{client_id_key}}={{client_id_value}}&{{client_secret_key}}={{client_secret_value}}&{{auth_code_key}}={{auth_code_value}}&{{redirect_uri_key}}={{{{redirect_uri_value}} | urlEncoder}}
2683
+ access_token_headers:
2684
+ title: Access Token Headers
2685
+ type: object
2686
+ additionalProperties: true
2687
+ description: |-
2688
+ The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
2689
+ examples:
2690
+ - {
2691
+ "Authorization": "Basic {{ {{ client_id_value }}:{{ client_secret_value }} | base64Encoder }}",
2692
+ }
2693
+ access_token_params:
2694
+ title: Access Token Query Params (Json Encoded)
2695
+ type: object
2696
+ additionalProperties: true
2697
+ description: |-
2698
+ The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
2699
+ When this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.
2700
+ examples:
2701
+ - {
2702
+ "{{ auth_code_key }}": "{{ auth_code_value }}",
2703
+ "{{ client_id_key }}": "{{ client_id_value }}",
2704
+ "{{ client_secret_key }}": "{{ client_secret_value }}",
2705
+ }
2706
+ extract_output:
2707
+ title: Extract Output
2708
+ type: array
2709
+ items:
2710
+ type: string
2711
+ description: |-
2712
+ The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.
2713
+ examples:
2714
+ - ["access_token", "refresh_token", "other_field"]
2715
+ state:
2716
+ title: Configurable State Query Param
2717
+ type: object
2718
+ additionalProperties: true
2719
+ required:
2720
+ - min
2721
+ - max
2722
+ description: |-
2723
+ The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,
2724
+ including length and complexity.
2725
+ properties:
2726
+ min:
2727
+ type: integer
2728
+ max:
2729
+ type: integer
2730
+ examples:
2731
+ - { "min": 7, "max": 128 }
2732
+ client_id_key:
2733
+ title: Client ID Key Override
2734
+ type: string
2735
+ description: |-
2736
+ The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.
2737
+ examples:
2738
+ - "my_custom_client_id_key_name"
2739
+ client_secret_key:
2740
+ title: Client Secret Key Override
2741
+ type: string
2742
+ description: |-
2743
+ The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.
2744
+ examples:
2745
+ - "my_custom_client_secret_key_name"
2746
+ scope_key:
2747
+ title: Scopes Key Override
2748
+ type: string
2749
+ description: |-
2750
+ The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.
2751
+ examples:
2752
+ - "my_custom_scope_key_key_name"
2753
+ state_key:
2754
+ title: State Key Override
2755
+ type: string
2756
+ description: |-
2757
+ The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.
2758
+ examples:
2759
+ - "my_custom_state_key_key_name"
2760
+ auth_code_key:
2761
+ title: Auth Code Key Override
2762
+ type: string
2763
+ description: |-
2764
+ The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.
2765
+ examples:
2766
+ - "my_custom_auth_code_key_name"
2767
+ redirect_uri_key:
2768
+ title: Redirect URI Key Override
2769
+ type: string
2770
+ description: |-
2771
+ The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.
2772
+ examples:
2773
+ - "my_custom_redirect_uri_key_name"
2774
+ complete_oauth_output_specification:
2775
+ title: "OAuth output specification"
2776
+ description: |-
2777
+ OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are
2778
+ returned by the distant OAuth APIs.
2779
+ Must be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.
2780
+ For each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,
2781
+ Examples:
2782
+ complete_oauth_output_specification={
2783
+ refresh_token: {
2784
+ type: string,
2785
+ path_in_connector_config: ['credentials', 'refresh_token']
2786
+ }
2787
+ }
2788
+ type: object
2789
+ additionalProperties: true
2790
+ examples:
2791
+ - refresh_token:
2792
+ type: string,
2793
+ path_in_connector_config: ["credentials", "refresh_token"]
2794
+ complete_oauth_server_input_specification:
2795
+ title: "OAuth input specification"
2796
+ description: |-
2797
+ OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.
2798
+ Must be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the
2799
+ server when completing an OAuth flow (typically exchanging an auth code for refresh token).
2800
+ Examples:
2801
+ complete_oauth_server_input_specification={
2802
+ client_id: {
2803
+ type: string
2804
+ },
2805
+ client_secret: {
2806
+ type: string
2807
+ }
2808
+ }
2809
+ type: object
2810
+ additionalProperties: true
2811
+ examples:
2812
+ - client_id:
2813
+ type: string
2814
+ client_secret:
2815
+ type: string
2816
+ complete_oauth_server_output_specification:
2817
+ title: "OAuth server output specification"
2818
+ description: |-
2819
+ OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that
2820
+ also need to be merged back into the connector configuration at runtime.
2821
+ This is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that
2822
+ are necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore
2823
+ they would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)
2824
+ Must be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the
2825
+ connector when using OAuth flow APIs.
2826
+ These fields are to be merged back to `ConnectorSpecification.connectionSpecification`.
2827
+ For each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,
2828
+ Examples:
2829
+ complete_oauth_server_output_specification={
2830
+ client_id: {
2831
+ type: string,
2832
+ path_in_connector_config: ['credentials', 'client_id']
2833
+ },
2834
+ client_secret: {
2835
+ type: string,
2836
+ path_in_connector_config: ['credentials', 'client_secret']
2837
+ }
2838
+ }
2839
+ type: object
2840
+ additionalProperties: true
2841
+ examples:
2842
+ - client_id:
2843
+ type: string,
2844
+ path_in_connector_config: ["credentials", "client_id"]
2845
+ client_secret:
2846
+ type: string,
2847
+ path_in_connector_config: ["credentials", "client_secret"]
2848
+ OffsetIncrement:
2849
+ title: Offset Increment
2850
+ description: Pagination strategy that returns the number of records reads so far and returns it as the next page token.
2851
+ type: object
2852
+ required:
2853
+ - type
2854
+ properties:
2855
+ type:
2856
+ type: string
2857
+ enum: [OffsetIncrement]
2858
+ page_size:
2859
+ title: Limit
2860
+ description: The number of records to include in each pages.
2861
+ anyOf:
2862
+ - type: integer
2863
+ - type: string
2864
+ interpolation_context:
2865
+ - config
2866
+ - response
2867
+ examples:
2868
+ - 100
2869
+ - "{{ config['page_size'] }}"
2870
+ inject_on_first_request:
2871
+ title: Inject Offset
2872
+ description: Using the `offset` with value `0` during the first request
2873
+ type: boolean
2874
+ default: false
2875
+ $parameters:
2876
+ type: object
2877
+ additionalProperties: true
2878
+ PageIncrement:
2879
+ title: Page Increment
2880
+ description: Pagination strategy that returns the number of pages reads so far and returns it as the next page token.
2881
+ type: object
2882
+ required:
2883
+ - type
2884
+ properties:
2885
+ type:
2886
+ type: string
2887
+ enum: [PageIncrement]
2888
+ page_size:
2889
+ title: Page Size
2890
+ description: The number of records to include in each pages.
2891
+ interpolation_context:
2892
+ - config
2893
+ anyOf:
2894
+ - type: integer
2895
+ - type: string
2896
+ examples:
2897
+ - 100
2898
+ - "100"
2899
+ - "{{ config['page_size'] }}"
2900
+ start_from_page:
2901
+ title: Start From Page
2902
+ description: Index of the first page to request.
2903
+ type: integer
2904
+ default: 0
2905
+ examples:
2906
+ - 0
2907
+ - 1
2908
+ inject_on_first_request:
2909
+ title: Inject Page Number
2910
+ description: Using the `page number` with value defined by `start_from_page` during the first request
2911
+ type: boolean
2912
+ default: false
2913
+ $parameters:
2914
+ type: object
2915
+ additionalProperties: true
2916
+ ParentStreamConfig:
2917
+ title: Parent Stream Config
2918
+ description: Describes how to construct partitions from the records retrieved from the parent stream..
2919
+ type: object
2920
+ required:
2921
+ - type
2922
+ - parent_key
2923
+ - partition_field
2924
+ - stream
2925
+ properties:
2926
+ type:
2927
+ type: string
2928
+ enum: [ParentStreamConfig]
2929
+ lazy_read_pointer:
2930
+ title: Lazy Read Pointer
2931
+ description: If set, this will enable lazy reading, using the initial read of parent records to extract child records.
2932
+ type: array
2933
+ default: [ ]
2934
+ items:
2935
+ - type: string
2936
+ interpolation_context:
2937
+ - config
2938
+ parent_key:
2939
+ title: Parent Key
2940
+ description: The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.
2941
+ type: string
2942
+ examples:
2943
+ - "id"
2944
+ - "{{ config['parent_record_id'] }}"
2945
+ stream:
2946
+ title: Parent Stream
2947
+ description: Reference to the parent stream.
2948
+ anyOf:
2949
+ - "$ref": "#/definitions/DeclarativeStream"
2950
+ - "$ref": "#/definitions/StateDelegatingStream"
2951
+ partition_field:
2952
+ title: Current Parent Key Value Identifier
2953
+ description: While iterating over parent records during a sync, the parent_key value can be referenced by using this field.
2954
+ type: string
2955
+ examples:
2956
+ - "parent_id"
2957
+ - "{{ config['parent_partition_field'] }}"
2958
+ request_option:
2959
+ title: Request Option
2960
+ description: A request option describing where the parent key value should be injected into and under what field name if applicable.
2961
+ "$ref": "#/definitions/RequestOption"
2962
+ incremental_dependency:
2963
+ title: Incremental Dependency
2964
+ description: Indicates whether the parent stream should be read incrementally based on updates in the child stream.
2965
+ type: boolean
2966
+ default: false
2967
+ extra_fields:
2968
+ title: Extra Fields
2969
+ description: Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.
2970
+ interpolation_context:
2971
+ - config
2972
+ type: array
2973
+ items:
2974
+ type: array
2975
+ items:
2976
+ type: string
2977
+ description: Defines a field path as an array of strings.
2978
+ examples:
2979
+ - ["field1"]
2980
+ - ["nested", "field2"]
2981
+ $parameters:
2982
+ type: object
2983
+ additionalProperties: true
2984
+ PrimaryKey:
2985
+ title: Primary Key
2986
+ description: The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.
2987
+ anyOf:
2988
+ - type: string
2989
+ - type: array
2990
+ items:
2991
+ type: string
2992
+ - type: array
2993
+ items:
2994
+ type: array
2995
+ items:
2996
+ type: string
2997
+ default: ""
2998
+ examples:
2999
+ - id
3000
+ - ["code", "type"]
3001
+ RecordFilter:
3002
+ title: Record Filter
3003
+ description: Filter applied on a list of records.
3004
+ type: object
3005
+ required:
3006
+ - type
3007
+ properties:
3008
+ type:
3009
+ type: string
3010
+ enum: [RecordFilter]
3011
+ condition:
3012
+ description: The predicate to filter a record. Records will be removed if evaluated to False.
3013
+ type: string
3014
+ default: ""
3015
+ interpolation_context:
3016
+ - config
3017
+ - next_page_token
3018
+ - record
3019
+ - stream_interval
3020
+ - stream_partition
3021
+ - stream_slice
3022
+ examples:
3023
+ - "{{ record['created_at'] >= stream_interval['start_time'] }}"
3024
+ - "{{ record.status in ['active', 'expired'] }}"
3025
+ $parameters:
3026
+ type: object
3027
+ additionalProperties: true
3028
+ RecordSelector:
3029
+ title: Record Selector
3030
+ description: Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering records based on a heuristic.
3031
+ type: object
3032
+ required:
3033
+ - type
3034
+ - extractor
3035
+ properties:
3036
+ type:
3037
+ type: string
3038
+ enum: [RecordSelector]
3039
+ extractor:
3040
+ anyOf:
3041
+ - "$ref": "#/definitions/CustomRecordExtractor"
3042
+ - "$ref": "#/definitions/DpathExtractor"
3043
+ record_filter:
3044
+ title: Record Filter
3045
+ description: Responsible for filtering records to be emitted by the Source.
3046
+ anyOf:
3047
+ - "$ref": "#/definitions/CustomRecordFilter"
3048
+ - "$ref": "#/definitions/RecordFilter"
3049
+ schema_normalization:
3050
+ title: Schema Normalization
3051
+ description: Responsible for normalization according to the schema.
3052
+ anyOf:
3053
+ - "$ref": "#/definitions/SchemaNormalization"
3054
+ - "$ref": "#/definitions/CustomSchemaNormalization"
3055
+ default: None
3056
+ transform_before_filtering:
3057
+ description: If true, transformation will be applied before record filtering.
3058
+ type: boolean
3059
+ default: false
3060
+ $parameters:
3061
+ type: object
3062
+ additionalProperties: true
3063
+ SchemaNormalization:
3064
+ title: Schema Normalization
3065
+ description: Responsible for normalization according to the schema.
3066
+ type: string
3067
+ enum:
3068
+ - None
3069
+ - Default
3070
+ examples:
3071
+ - None
3072
+ - Default
3073
+ RemoveFields:
3074
+ title: Remove Fields
3075
+ description: A transformation which removes fields from a record. The fields removed are designated using FieldPointers. During transformation, if a field or any of its parents does not exist in the record, no error is thrown.
3076
+ type: object
3077
+ required:
3078
+ - type
3079
+ - field_pointers
3080
+ properties:
3081
+ type:
3082
+ type: string
3083
+ enum: [RemoveFields]
3084
+ condition:
3085
+ description: The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,
3086
+ type: string
3087
+ default: ""
3088
+ interpolation_context:
3089
+ - config
3090
+ - property
3091
+ - parameters
3092
+ examples:
3093
+ - "{{ property|string == '' }}"
3094
+ - "{{ property is integer }}"
3095
+ - "{{ property|length > 5 }}"
3096
+ - "{{ property == 'some_string_to_match' }}"
3097
+ field_pointers:
3098
+ title: Field Paths
3099
+ description: Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.
3100
+ type: array
3101
+ items:
3102
+ items:
3103
+ type: string
3104
+ examples:
3105
+ - ["tags"]
3106
+ - [["content", "html"], ["content", "plain_text"]]
3107
+ RequestPath:
3108
+ title: Request Path
3109
+ description: Specifies where in the request path a component's value should be inserted.
3110
+ type: object
3111
+ required:
3112
+ - type
3113
+ properties:
3114
+ type:
3115
+ type: string
3116
+ enum: [RequestPath]
3117
+ RequestOption:
3118
+ title: Request Option
3119
+ description: Specifies the key field or path and where in the request a component's value should be injected.
3120
+ type: object
3121
+ required:
3122
+ - type
3123
+ - inject_into
3124
+ properties:
3125
+ type:
3126
+ type: string
3127
+ enum: [RequestOption]
3128
+ field_name:
3129
+ title: Field Name
3130
+ description: Configures which key should be used in the location that the descriptor is being injected into. We hope to eventually deprecate this field in favor of `field_path` for all request_options, but must currently maintain it for backwards compatibility in the Builder.
3131
+ type: string
3132
+ examples:
3133
+ - segment_id
3134
+ interpolation_context:
3135
+ - config
3136
+ - parameters
3137
+ field_path:
3138
+ title: Field Path
3139
+ description: Configures a path to be used for nested structures in JSON body requests (e.g. GraphQL queries)
3140
+ type: array
3141
+ items:
3142
+ type: string
3143
+ examples:
3144
+ - ["data", "viewer", "id"]
3145
+ interpolation_context:
3146
+ - config
3147
+ - parameters
3148
+ inject_into:
3149
+ title: Inject Into
3150
+ description: Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.
3151
+ enum:
3152
+ - request_parameter
3153
+ - header
3154
+ - body_data
3155
+ - body_json
3156
+ examples:
3157
+ - request_parameter
3158
+ - header
3159
+ - body_data
3160
+ - body_json
3161
+ Schemas:
3162
+ title: Schemas
3163
+ description: The stream schemas representing the shape of the data emitted by the stream.
3164
+ type: object
3165
+ additionalProperties: true
3166
+ LegacySessionTokenAuthenticator:
3167
+ title: Session Token Authenticator
3168
+ description: Deprecated - use SessionTokenAuthenticator instead. Authenticator for requests authenticated using session tokens. A session token is a random value generated by a server to identify a specific user for the duration of one interaction session.
3169
+ type: object
3170
+ required:
3171
+ - type
3172
+ - header
3173
+ - login_url
3174
+ - session_token_response_key
3175
+ - validate_session_url
3176
+ properties:
3177
+ type:
3178
+ type: string
3179
+ enum: [LegacySessionTokenAuthenticator]
3180
+ header:
3181
+ title: Session Request Header
3182
+ description: The name of the session token header that will be injected in the request
3183
+ type: string
3184
+ examples:
3185
+ - "X-Session"
3186
+ login_url:
3187
+ title: Login Path
3188
+ description: Path of the login URL (do not include the base URL)
3189
+ type: string
3190
+ examples:
3191
+ - session
3192
+ session_token:
3193
+ title: Session Token
3194
+ description: Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair
3195
+ type: string
3196
+ example:
3197
+ - "{{ config['session_token'] }}"
3198
+ session_token_response_key:
3199
+ title: Response Token Response Key
3200
+ description: Name of the key of the session token to be extracted from the response
3201
+ type: string
3202
+ examples:
3203
+ - id
3204
+ username:
3205
+ title: Username
3206
+ description: Username used to authenticate and obtain a session token
3207
+ type: string
3208
+ examples:
3209
+ - " {{ config['username'] }}"
3210
+ password:
3211
+ title: Password
3212
+ description: Password used to authenticate and obtain a session token
3213
+ type: string
3214
+ default: ""
3215
+ examples:
3216
+ - "{{ config['password'] }}"
3217
+ - ""
3218
+ validate_session_url:
3219
+ title: Validate Session Path
3220
+ description: Path of the URL to use to validate that the session token is valid (do not include the base URL)
3221
+ type: string
3222
+ examples:
3223
+ - "user/current"
3224
+ $parameters:
3225
+ type: object
3226
+ additionalProperties: true
3227
+ StateDelegatingStream:
3228
+ description: (This component is experimental. Use at your own risk.) Orchestrate the retriever's usage based on the state value.
3229
+ type: object
3230
+ required:
3231
+ - type
3232
+ - name
3233
+ - full_refresh_stream
3234
+ - incremental_stream
3235
+ properties:
3236
+ type:
3237
+ type: string
3238
+ enum: [ StateDelegatingStream ]
3239
+ name:
3240
+ title: Name
3241
+ description: The stream name.
3242
+ type: string
3243
+ default: ""
3244
+ example:
3245
+ - "Users"
3246
+ full_refresh_stream:
3247
+ title: Retriever
3248
+ description: Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.
3249
+ "$ref": "#/definitions/DeclarativeStream"
3250
+ incremental_stream:
3251
+ title: Retriever
3252
+ description: Component used to coordinate how records are extracted across stream slices and request pages when the state provided.
3253
+ "$ref": "#/definitions/DeclarativeStream"
3254
+ $parameters:
3255
+ type: object
3256
+ additionalProperties: true
3257
+ SimpleRetriever:
3258
+ description: Retrieves records by synchronously sending requests to fetch records. The retriever acts as an orchestrator between the requester, the record selector, the paginator, and the partition router.
3259
+ type: object
3260
+ required:
3261
+ - type
3262
+ - record_selector
3263
+ - requester
3264
+ properties:
3265
+ type:
3266
+ type: string
3267
+ enum: [SimpleRetriever]
3268
+ record_selector:
3269
+ description: Component that describes how to extract records from a HTTP response.
3270
+ "$ref": "#/definitions/RecordSelector"
3271
+ requester:
3272
+ description: Requester component that describes how to prepare HTTP requests to send to the source API.
3273
+ anyOf:
3274
+ - "$ref": "#/definitions/CustomRequester"
3275
+ - "$ref": "#/definitions/HttpRequester"
3276
+ paginator:
3277
+ description: Paginator component that describes how to navigate through the API's pages.
3278
+ anyOf:
3279
+ - "$ref": "#/definitions/DefaultPaginator"
3280
+ - "$ref": "#/definitions/NoPagination"
3281
+ ignore_stream_slicer_parameters_on_paginated_requests:
3282
+ description: If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.
3283
+ type: boolean
3284
+ default: false
3285
+ partition_router:
3286
+ title: Partition Router
3287
+ description: PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.
3288
+ default: []
3289
+ anyOf:
3290
+ - "$ref": "#/definitions/CustomPartitionRouter"
3291
+ - "$ref": "#/definitions/ListPartitionRouter"
3292
+ - "$ref": "#/definitions/SubstreamPartitionRouter"
3293
+ - type: array
3294
+ items:
3295
+ anyOf:
3296
+ - "$ref": "#/definitions/CustomPartitionRouter"
3297
+ - "$ref": "#/definitions/ListPartitionRouter"
3298
+ - "$ref": "#/definitions/SubstreamPartitionRouter"
3299
+ decoder:
3300
+ title: Decoder
3301
+ description: Component decoding the response so records can be extracted.
3302
+ anyOf:
3303
+ - "$ref": "#/definitions/CustomDecoder"
3304
+ - "$ref": "#/definitions/CsvDecoder"
3305
+ - "$ref": "#/definitions/GzipDecoder"
3306
+ - "$ref": "#/definitions/JsonDecoder"
3307
+ - "$ref": "#/definitions/JsonlDecoder"
3308
+ - "$ref": "#/definitions/IterableDecoder"
3309
+ - "$ref": "#/definitions/XmlDecoder"
3310
+ - "$ref": "#/definitions/ZipfileDecoder"
3311
+ $parameters:
3312
+ type: object
3313
+ additionalProperties: true
3314
+ GzipDecoder:
3315
+ type: object
3316
+ required:
3317
+ - type
3318
+ - decoder
3319
+ properties:
3320
+ type:
3321
+ type: string
3322
+ enum: [GzipDecoder]
3323
+ decoder:
3324
+ anyOf:
3325
+ - "$ref": "#/definitions/CsvDecoder"
3326
+ - "$ref": "#/definitions/GzipDecoder"
3327
+ - "$ref": "#/definitions/JsonDecoder"
3328
+ - "$ref": "#/definitions/JsonlDecoder"
3329
+ CsvDecoder:
3330
+ type: object
3331
+ required:
3332
+ - type
3333
+ properties:
3334
+ type:
3335
+ type: string
3336
+ enum: [CsvDecoder]
3337
+ encoding:
3338
+ type: string
3339
+ default: utf-8
3340
+ delimiter:
3341
+ type: string
3342
+ default: ","
3343
+ AsyncJobStatusMap:
3344
+ description: Matches the api job status to Async Job Status.
3345
+ type: object
3346
+ required:
3347
+ - running
3348
+ - completed
3349
+ - failed
3350
+ - timeout
3351
+ properties:
3352
+ type:
3353
+ type: string
3354
+ enum: [AsyncJobStatusMap]
3355
+ running:
3356
+ type: array
3357
+ items:
3358
+ type: string
3359
+ completed:
3360
+ type: array
3361
+ items:
3362
+ type: string
3363
+ failed:
3364
+ type: array
3365
+ items:
3366
+ type: string
3367
+ timeout:
3368
+ type: array
3369
+ items:
3370
+ type: string
3371
+ AsyncRetriever:
3372
+ description: "Retrieves records by Asynchronously sending requests to fetch records. The retriever acts as an orchestrator between the requester, the record selector, the paginator, and the partition router."
3373
+ type: object
3374
+ required:
3375
+ - type
3376
+ - record_selector
3377
+ - status_mapping
3378
+ - creation_requester
3379
+ - polling_requester
3380
+ - download_requester
3381
+ - status_extractor
3382
+ - download_target_extractor
3383
+ properties:
3384
+ type:
3385
+ type: string
3386
+ enum: [AsyncRetriever]
3387
+ record_selector:
3388
+ description: Component that describes how to extract records from a HTTP response.
3389
+ "$ref": "#/definitions/RecordSelector"
3390
+ status_mapping:
3391
+ description: Async Job Status to Airbyte CDK Async Job Status mapping.
3392
+ anyOf:
3393
+ - "$ref": "#/definitions/AsyncJobStatusMap"
3394
+ status_extractor:
3395
+ description: Responsible for fetching the actual status of the async job.
3396
+ anyOf:
3397
+ - "$ref": "#/definitions/CustomRecordExtractor"
3398
+ - "$ref": "#/definitions/DpathExtractor"
3399
+ download_target_extractor:
3400
+ description: Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.
3401
+ anyOf:
3402
+ - "$ref": "#/definitions/CustomRecordExtractor"
3403
+ - "$ref": "#/definitions/DpathExtractor"
3404
+ download_extractor:
3405
+ description: Responsible for fetching the records from provided urls.
3406
+ anyOf:
3407
+ - "$ref": "#/definitions/CustomRecordExtractor"
3408
+ - "$ref": "#/definitions/DpathExtractor"
3409
+ - "$ref": "#/definitions/ResponseToFileExtractor"
3410
+ creation_requester:
3411
+ description: Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.
3412
+ anyOf:
3413
+ - "$ref": "#/definitions/CustomRequester"
3414
+ - "$ref": "#/definitions/HttpRequester"
3415
+ polling_requester:
3416
+ description: Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.
3417
+ anyOf:
3418
+ - "$ref": "#/definitions/CustomRequester"
3419
+ - "$ref": "#/definitions/HttpRequester"
3420
+ polling_job_timeout:
3421
+ description: The time in minutes after which the single Async Job should be considered as Timed Out.
3422
+ anyOf:
3423
+ - type: integer
3424
+ - type: string
3425
+ interpolation_context:
3426
+ - config
3427
+ download_target_requester:
3428
+ description: Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.
3429
+ anyOf:
3430
+ - "$ref": "#/definitions/CustomRequester"
3431
+ - "$ref": "#/definitions/HttpRequester"
3432
+ download_requester:
3433
+ description: Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.
3434
+ anyOf:
3435
+ - "$ref": "#/definitions/CustomRequester"
3436
+ - "$ref": "#/definitions/HttpRequester"
3437
+ download_paginator:
3438
+ description: Paginator component that describes how to navigate through the API's pages during download.
3439
+ anyOf:
3440
+ - "$ref": "#/definitions/DefaultPaginator"
3441
+ - "$ref": "#/definitions/NoPagination"
3442
+ abort_requester:
3443
+ description: Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.
3444
+ anyOf:
3445
+ - "$ref": "#/definitions/CustomRequester"
3446
+ - "$ref": "#/definitions/HttpRequester"
3447
+ delete_requester:
3448
+ description: Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.
3449
+ anyOf:
3450
+ - "$ref": "#/definitions/CustomRequester"
3451
+ - "$ref": "#/definitions/HttpRequester"
3452
+ partition_router:
3453
+ title: Partition Router
3454
+ description: PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.
3455
+ default: []
3456
+ anyOf:
3457
+ - "$ref": "#/definitions/CustomPartitionRouter"
3458
+ - "$ref": "#/definitions/ListPartitionRouter"
3459
+ - "$ref": "#/definitions/SubstreamPartitionRouter"
3460
+ - type: array
3461
+ items:
3462
+ anyOf:
3463
+ - "$ref": "#/definitions/CustomPartitionRouter"
3464
+ - "$ref": "#/definitions/ListPartitionRouter"
3465
+ - "$ref": "#/definitions/SubstreamPartitionRouter"
3466
+ decoder:
3467
+ title: Decoder
3468
+ description: Component decoding the response so records can be extracted.
3469
+ anyOf:
3470
+ - "$ref": "#/definitions/CustomDecoder"
3471
+ - "$ref": "#/definitions/CsvDecoder"
3472
+ - "$ref": "#/definitions/GzipDecoder"
3473
+ - "$ref": "#/definitions/JsonDecoder"
3474
+ - "$ref": "#/definitions/JsonlDecoder"
3475
+ - "$ref": "#/definitions/IterableDecoder"
3476
+ - "$ref": "#/definitions/XmlDecoder"
3477
+ - "$ref": "#/definitions/ZipfileDecoder"
3478
+ download_decoder:
3479
+ title: Download Decoder
3480
+ description: Component decoding the download response so records can be extracted.
3481
+ anyOf:
3482
+ - "$ref": "#/definitions/CustomDecoder"
3483
+ - "$ref": "#/definitions/CsvDecoder"
3484
+ - "$ref": "#/definitions/GzipDecoder"
3485
+ - "$ref": "#/definitions/JsonDecoder"
3486
+ - "$ref": "#/definitions/JsonlDecoder"
3487
+ - "$ref": "#/definitions/IterableDecoder"
3488
+ - "$ref": "#/definitions/XmlDecoder"
3489
+ - "$ref": "#/definitions/ZipfileDecoder"
3490
+ $parameters:
3491
+ type: object
3492
+ additionalProperties: true
3493
+ Spec:
3494
+ title: Spec
3495
+ description: A source specification made up of connector metadata and how it can be configured.
3496
+ type: object
3497
+ required:
3498
+ - type
3499
+ - connection_specification
3500
+ properties:
3501
+ type:
3502
+ type: string
3503
+ enum: [Spec]
3504
+ connection_specification:
3505
+ title: Connection Specification
3506
+ description: A connection specification describing how a the connector can be configured.
3507
+ type: object
3508
+ additionalProperties: true
3509
+ documentation_url:
3510
+ title: Documentation URL
3511
+ description: URL of the connector's documentation page.
3512
+ type: string
3513
+ examples:
3514
+ - "https://docs.airbyte.com/integrations/sources/dremio"
3515
+ advanced_auth:
3516
+ title: Advanced Auth
3517
+ description: Advanced specification for configuring the authentication flow.
3518
+ "$ref": "#/definitions/AuthFlow"
3519
+ SubstreamPartitionRouter:
3520
+ title: Substream Partition Router
3521
+ description: Partition router that is used to retrieve records that have been partitioned according to records from the specified parent streams. An example of a parent stream is automobile brands and the substream would be the various car models associated with each branch.
3522
+ type: object
3523
+ required:
3524
+ - type
3525
+ - parent_stream_configs
3526
+ properties:
3527
+ type:
3528
+ type: string
3529
+ enum: [SubstreamPartitionRouter]
3530
+ parent_stream_configs:
3531
+ title: Parent Stream Configs
3532
+ description: Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.
3533
+ type: array
3534
+ items:
3535
+ "$ref": "#/definitions/ParentStreamConfig"
3536
+ $parameters:
3537
+ type: object
3538
+ additionalProperties: true
3539
+ ValueType:
3540
+ title: Value Type
3541
+ description: A schema type.
3542
+ type: string
3543
+ enum:
3544
+ - string
3545
+ - number
3546
+ - integer
3547
+ - boolean
3548
+ WaitTimeFromHeader:
3549
+ title: Wait Time Extracted From Response Header
3550
+ description: Extract wait time from a HTTP header in the response.
3551
+ type: object
3552
+ required:
3553
+ - type
3554
+ - header
3555
+ properties:
3556
+ type:
3557
+ type: string
3558
+ enum: [WaitTimeFromHeader]
3559
+ header:
3560
+ title: Response Header Name
3561
+ description: The name of the response header defining how long to wait before retrying.
3562
+ type: string
3563
+ interpolation_context:
3564
+ - config
3565
+ examples:
3566
+ - "Retry-After"
3567
+ regex:
3568
+ title: Extraction Regex
3569
+ description: Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.
3570
+ type: string
3571
+ examples:
3572
+ - "([-+]?\\d+)"
3573
+ max_waiting_time_in_seconds:
3574
+ title: Max Waiting Time in Seconds
3575
+ description: Given the value extracted from the header is greater than this value, stop the stream.
3576
+ type: number
3577
+ examples:
3578
+ - 3600
3579
+ $parameters:
3580
+ type: object
3581
+ additionalProperties: true
3582
+ WaitUntilTimeFromHeader:
3583
+ title: Wait Until Time Defined In Response Header
3584
+ description: Extract time at which we can retry the request from response header and wait for the difference between now and that time.
3585
+ type: object
3586
+ required:
3587
+ - type
3588
+ - header
3589
+ properties:
3590
+ type:
3591
+ type: string
3592
+ enum: [WaitUntilTimeFromHeader]
3593
+ header:
3594
+ title: Response Header
3595
+ description: The name of the response header defining how long to wait before retrying.
3596
+ type: string
3597
+ interpolation_context:
3598
+ - config
3599
+ examples:
3600
+ - wait_time
3601
+ min_wait:
3602
+ title: Minimum Wait Time
3603
+ description: Minimum time to wait before retrying.
3604
+ anyOf:
3605
+ - type: number
3606
+ - type: string
3607
+ interpolation_context:
3608
+ - config
3609
+ examples:
3610
+ - 10
3611
+ - "60"
3612
+ regex:
3613
+ title: Extraction Regex
3614
+ description: Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.
3615
+ type: string
3616
+ interpolation_context:
3617
+ - config
3618
+ examples:
3619
+ - "([-+]?\\d+)"
3620
+ $parameters:
3621
+ type: object
3622
+ additionalProperties: true
3623
+ ComponentMappingDefinition:
3624
+ title: Component Mapping Definition
3625
+ description: (This component is experimental. Use at your own risk.) Specifies a mapping definition to update or add fields in a record or configuration. This allows dynamic mapping of data by interpolating values into the template based on provided contexts.
3626
+ type: object
3627
+ required:
3628
+ - type
3629
+ - field_path
3630
+ - value
3631
+ properties:
3632
+ type:
3633
+ type: string
3634
+ enum: [ComponentMappingDefinition]
3635
+ field_path:
3636
+ title: Field Path
3637
+ description: A list of potentially nested fields indicating the full path where value will be added or updated.
3638
+ type: array
3639
+ items:
3640
+ type: string
3641
+ interpolation_context:
3642
+ - config
3643
+ - components_values
3644
+ - stream_slice
3645
+ - stream_template_config
3646
+ examples:
3647
+ - ["data"]
3648
+ - ["data", "records"]
3649
+ - ["data", 1, "name"]
3650
+ - ["data", "{{ components_values.name }}"]
3651
+ - ["data", "*", "record"]
3652
+ - ["*", "**", "name"]
3653
+ value:
3654
+ title: Value
3655
+ description: The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.
3656
+ type: string
3657
+ interpolation_context:
3658
+ - config
3659
+ - stream_template_config
3660
+ - components_values
3661
+ - stream_slice
3662
+ examples:
3663
+ - "{{ components_values['updates'] }}"
3664
+ - "{{ components_values['MetaData']['LastUpdatedTime'] }}"
3665
+ - "{{ config['segment_id'] }}"
3666
+ - "{{ stream_slice['parent_id'] }}"
3667
+ - "{{ stream_slice['extra_fields']['name'] }}"
3668
+ value_type:
3669
+ title: Value Type
3670
+ description: The expected data type of the value. If omitted, the type will be inferred from the value provided.
3671
+ "$ref": "#/definitions/ValueType"
3672
+ $parameters:
3673
+ type: object
3674
+ additionalProperties: true
3675
+ HttpComponentsResolver:
3676
+ type: object
3677
+ description: (This component is experimental. Use at your own risk.) Component resolve and populates stream templates with components fetched via an HTTP retriever.
3678
+ properties:
3679
+ type:
3680
+ type: string
3681
+ enum: [HttpComponentsResolver]
3682
+ retriever:
3683
+ title: Retriever
3684
+ description: Component used to coordinate how records are extracted across stream slices and request pages.
3685
+ anyOf:
3686
+ - "$ref": "#/definitions/AsyncRetriever"
3687
+ - "$ref": "#/definitions/CustomRetriever"
3688
+ - "$ref": "#/definitions/SimpleRetriever"
3689
+ components_mapping:
3690
+ type: array
3691
+ items:
3692
+ "$ref": "#/definitions/ComponentMappingDefinition"
3693
+ $parameters:
3694
+ type: object
3695
+ additionalProperties: true
3696
+ required:
3697
+ - type
3698
+ - retriever
3699
+ - components_mapping
3700
+ StreamConfig:
3701
+ title: Stream Config
3702
+ description: (This component is experimental. Use at your own risk.) Describes how to get streams config from the source config.
3703
+ type: object
3704
+ required:
3705
+ - type
3706
+ - configs_pointer
3707
+ properties:
3708
+ type:
3709
+ type: string
3710
+ enum: [StreamConfig]
3711
+ configs_pointer:
3712
+ title: Configs Pointer
3713
+ description: A list of potentially nested fields indicating the full path in source config file where streams configs located.
3714
+ type: array
3715
+ items:
3716
+ type: string
3717
+ interpolation_context:
3718
+ - parameters
3719
+ examples:
3720
+ - ["data"]
3721
+ - ["data", "streams"]
3722
+ - ["data", "{{ parameters.name }}"]
3723
+ $parameters:
3724
+ type: object
3725
+ additionalProperties: true
3726
+ ConfigComponentsResolver:
3727
+ type: object
3728
+ description: (This component is experimental. Use at your own risk.) Resolves and populates stream templates with components fetched from the source config.
3729
+ properties:
3730
+ type:
3731
+ type: string
3732
+ enum: [ConfigComponentsResolver]
3733
+ stream_config:
3734
+ "$ref": "#/definitions/StreamConfig"
3735
+ components_mapping:
3736
+ type: array
3737
+ items:
3738
+ "$ref": "#/definitions/ComponentMappingDefinition"
3739
+ $parameters:
3740
+ type: object
3741
+ additionalProperties: true
3742
+ required:
3743
+ - type
3744
+ - stream_config
3745
+ - components_mapping
3746
+ DynamicDeclarativeStream:
3747
+ type: object
3748
+ description: (This component is experimental. Use at your own risk.) A component that described how will be created declarative streams based on stream template.
3749
+ properties:
3750
+ type:
3751
+ type: string
3752
+ enum: [DynamicDeclarativeStream]
3753
+ stream_template:
3754
+ title: Stream Template
3755
+ description: Reference to the stream template.
3756
+ "$ref": "#/definitions/DeclarativeStream"
3757
+ components_resolver:
3758
+ title: Components Resolver
3759
+ description: Component resolve and populates stream templates with components values.
3760
+ anyOf:
3761
+ - "$ref": "#/definitions/HttpComponentsResolver"
3762
+ - "$ref": "#/definitions/ConfigComponentsResolver"
3763
+ required:
3764
+ - type
3765
+ - stream_template
3766
+ - components_resolver
3767
+ interpolation:
3768
+ variables:
3769
+ - title: config
3770
+ description: The connector configuration. The object's keys are the same as the the keys defined in the connection specification.
3771
+ type: object
3772
+ examples:
3773
+ - start_date: 2010-01-01
3774
+ api_key: "*****"
3775
+ - title: parameters
3776
+ description: Additional runtime parameters, to be used for string interpolation. Parameters can be passed down from a parent component to its subcomponents using the $parameters key. This can be used to avoid repetitions.
3777
+ type: object
3778
+ examples:
3779
+ - path: "automations"
3780
+ data_export_path: "automations"
3781
+ cursor_field: "updated_at"
3782
+ - title: headers
3783
+ description: The HTTP headers from the last response received from the API. The object's keys are the header names from the response.
3784
+ type: object
3785
+ examples:
3786
+ - Server: nginx
3787
+ Date: Mon, 24 Apr 2023 20:17:21 GMT
3788
+ Content-Type: application/json
3789
+ Content-Length: "420"
3790
+ Connection: keep-alive
3791
+ referrer-policy: strict-origin-when-cross-origin
3792
+ x-content-type-options: nosniff
3793
+ x-ratelimit-limit: "600"
3794
+ x-ratelimit-remaining: "598"
3795
+ x-ratelimit-reset: "39"
3796
+ - title: last_record
3797
+ description: Last record extracted from the response received from the API.
3798
+ type: object
3799
+ examples:
3800
+ - name: "Test List: 19"
3801
+ id: 0236d6d2
3802
+ contact_count: 20
3803
+ _metadata:
3804
+ self: https://api.sendgrid.com/v3/marketing/lists/0236d6d2
3805
+ - title: last_page_size
3806
+ description: Number of records extracted from the last response received from the API.
3807
+ type: object
3808
+ examples:
3809
+ - 2
3810
+ - title: next_page_token
3811
+ description: Object describing the token to fetch the next page of records. The object has a single key "next_page_token".
3812
+ type: object
3813
+ examples:
3814
+ - next_page_token: 3
3815
+ - next_page_token: https://api.sendgrid.com/v3/marketing/lists/0236d6d2-75d2-42c5-962d-603e0deaf8d1
3816
+ - title: record
3817
+ description: The record being processed. The object's keys are the same keys as the records produced by the RecordSelector.
3818
+ type: object
3819
+ - title: response
3820
+ description: The body of the last response received from the API. The object's keys are the same keys as the response body's.
3821
+ type: object
3822
+ examples:
3823
+ - result:
3824
+ - name: "Test List: 19"
3825
+ id: 0236d6d2-75d2-42c5-962d-603e0deaf8d1
3826
+ contact_count: 20
3827
+ _metadata:
3828
+ self: https://api.sendgrid.com/v3/marketing/lists/0236d6d2
3829
+ _metadata:
3830
+ self: https://api.sendgrid.com/v3/marketing/lists?page_size=1&page_token=
3831
+ next: https://api.sendgrid.com/v3/marketing/lists?page_size=1&page_token=0236d6d2
3832
+ count: 82
3833
+ - title: creation_response
3834
+ description: The response received from the creation_requester in the AsyncRetriever component.
3835
+ type: object
3836
+ examples:
3837
+ - id: "1234"
3838
+ - title: polling_response
3839
+ description: The response received from the polling_requester in the AsyncRetriever component.
3840
+ type: object
3841
+ examples:
3842
+ - id: "1234"
3843
+ - title: download_target
3844
+ description: The `URL` received from the polling_requester in the AsyncRetriever with jobStatus as `COMPLETED`.
3845
+ type: string
3846
+ examples:
3847
+ - "https://api.sendgrid.com/v3/marketing/lists?page_size=1&page_token=0236d6d2&filename=xxx_yyy_zzz.csv"
3848
+ - title: stream_interval
3849
+ description: The current stream interval being processed. The keys are defined by the incremental sync component. Default keys are `start_time` and `end_time`.
3850
+ type: object
3851
+ examples:
3852
+ - start_time: "2020-01-01 00:00:00.000+00:00"
3853
+ end_time: "2020-01-02 00:00:00.000+00:00"
3854
+ - title: stream_partition
3855
+ description: The current stream partition being processed. The keys are defined by the partition router component.
3856
+ type: object
3857
+ examples:
3858
+ - survey_id: 1234
3859
+ - strategy: DESKTOP
3860
+ - survey_id: 1234
3861
+ strategy: MOBILE
3862
+ - title: stream_slice
3863
+ description: This variable is deprecated. Use stream_interval or stream_partition instead.
3864
+ type: object
3865
+ macros:
3866
+ - title: now_utc
3867
+ description: Returns the current date and time in the UTC timezone.
3868
+ arguments: {}
3869
+ return_type: Datetime
3870
+ examples:
3871
+ - "'{{ now_utc() }}' -> '2021-09-01 00:00:00+00:00'"
3872
+ - "'{{ now_utc().strftime('%Y-%m-%d') }}' -> '2021-09-01'"
3873
+ - title: today_utc
3874
+ description: Returns the current date in UTC timezone. The output is a date object.
3875
+ arguments: {}
3876
+ return_type: Date
3877
+ examples:
3878
+ - "'{{ today_utc() }}' -> '2021-09-01'"
3879
+ - "'{{ today_utc().strftime('%Y/%m/%d')}}' -> '2021/09/01'"
3880
+ - title: timestamp
3881
+ description: Converts a number or a string representing a datetime (formatted as ISO8601) to a timestamp. If the input is a number, it is converted to an int. If no timezone is specified, the string is interpreted as UTC.
3882
+ arguments:
3883
+ datetime: A string formatted as ISO8601 or an integer representing a unix timestamp
3884
+ return_type: int
3885
+ examples:
3886
+ - "'{{ timestamp(1646006400) }}' -> 1646006400"
3887
+ - "'{{ timestamp('2022-02-28') }}' -> 1646006400"
3888
+ - "'{{ timestamp('2022-02-28T00:00:00Z') }}' -> 1646006400"
3889
+ - "'{{ timestamp('2022-02-28 00:00:00Z') }}' -> 1646006400"
3890
+ - "'{{ timestamp('2022-02-28T00:00:00-08:00') }}' -> 1646035200"
3891
+ - title: max
3892
+ description: Returns the largest object of a iterable, or or two or more arguments.
3893
+ arguments:
3894
+ args: iterable or a sequence of two or more arguments
3895
+ return_type: Any
3896
+ examples:
3897
+ - "'{{ max(2, 3) }}' -> 3"
3898
+ - "'{{ max([2, 3]) }}' -> 3"
3899
+ - title: day_delta
3900
+ description: Returns the datetime of now() + num_days.
3901
+ arguments:
3902
+ num_days: The number of days to add to now
3903
+ format: How to format the output string
3904
+ return_type: str
3905
+ examples:
3906
+ - "'{{ day_delta(1) }}' -> '2021-09-02T00:00:00.000000+0000'"
3907
+ - "'{{ day_delta(-1) }}' -> '2021-08-31:00:00.000000+0000'"
3908
+ - "'{{ day_delta(25, format='%Y-%m-%d') }}' -> '2021-09-02'"
3909
+ - title: duration
3910
+ description: Converts an ISO8601 duration to datetime timedelta.
3911
+ arguments:
3912
+ duration_string: "A string representing an ISO8601 duration. See https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm for more details."
3913
+ return_type: datetime.timedelta
3914
+ examples:
3915
+ - "'{{ duration('P1D') }}' -> '1 day, 0:00:00'"
3916
+ - "'{{ duration('P6DT23H') }}' -> '6 days, 23:00:00'"
3917
+ - "'{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}' -> '2021-08-31T00:00:00Z'"
3918
+ - title: format_datetime
3919
+ description: Converts a datetime or a datetime-string to the specified format.
3920
+ arguments:
3921
+ datetime: The datetime object or a string to convert. If datetime is a string, it must be formatted as ISO8601.
3922
+ format: The datetime format.
3923
+ input_format: (optional) The datetime format in the case it is an string.
3924
+ return_type: str
3925
+ examples:
3926
+ - "{{ format_datetime(config['start_time'], '%Y-%m-%d') }}"
3927
+ - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ') }}"
3928
+ - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ', '%a, %d %b %Y %H:%M:%S %z') }}"
3929
+ - title: str_to_datetime
3930
+ description: Converts a string to a datetime object with UTC timezone.
3931
+ arguments:
3932
+ s: The string to convert.
3933
+ return_type: datetime.datetime
3934
+ examples:
3935
+ - "{{ str_to_datetime('2022-01-14') }}"
3936
+ - "{{ str_to_datetime('2022-01-01 13:45:30') }}"
3937
+ - "{{ str_to_datetime('2022-01-01T13:45:30+00:00') }}"
3938
+ - "{{ str_to_datetime('2022-01-01T13:45:30.123456Z') }}"
3939
+ filters:
3940
+ - title: hash
3941
+ description: Convert the specified value to a hashed string.
3942
+ arguments:
3943
+ hash_type: Valid hash type for converts ('md5' as default value).
3944
+ salt: An additional value to further protect sensitive data.
3945
+ return_type: str
3946
+ examples:
3947
+ - "{{ 'Test client_secret' | hash() }} -> '3032d57a12f76b61a820e47b9a5a0cbb'"
3948
+ - "{{ 'Test client_secret' | hash('md5') }} -> '3032d57a12f76b61a820e47b9a5a0cbb'"
3949
+ - "{{ 'Test client_secret' | hash('md5', salt='salt') }} -> '5011a0168579c2d94cbbe1c6ad14327c'"
3950
+ - title: base64encode
3951
+ description: Convert the specified value to a string in the base64 format.
3952
+ arguments: {}
3953
+ return_type: str
3954
+ examples:
3955
+ - "{{ 'Test client_secret' | base64encode }} -> 'VGVzdCBjbGllbnRfc2VjcmV0'"
3956
+ - title: base64decode
3957
+ description: Decodes the specified base64 format value into a common string.
3958
+ arguments: {}
3959
+ return_type: str
3960
+ examples:
3961
+ - "{{ 'ZmFrZSByZWZyZXNoX3Rva2VuIHZhbHVl' | base64decode }} -> 'fake refresh_token value'"
3962
+ - title: string
3963
+ description: Converts the specified value to a string.
3964
+ arguments: {}
3965
+ return_type: str
3966
+ examples:
3967
+ - '{{ 1 | string }} -> "1"'
3968
+ - '{{ ["hello", "world" | string }} -> "["hello", "world"]"'
3969
+ - title: regex_search
3970
+ description: Match the input string against a regular expression and return the first match.
3971
+ arguments:
3972
+ regex: The regular expression to search for. It must include a capture group.
3973
+ return_type: str
3974
+ examples:
3975
+ - '{{ "goodbye, cruel world" | regex_search("goodbye,\s(.*)$") }} -> "cruel world"'