airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- airbyte_cdk/__init__.py +355 -6
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +29 -10
- airbyte_cdk/connector.py +24 -24
- airbyte_cdk/connector_builder/README.md +53 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
- airbyte_cdk/connector_builder/main.py +45 -13
- airbyte_cdk/connector_builder/message_grouper.py +189 -50
- airbyte_cdk/connector_builder/models.py +3 -2
- airbyte_cdk/destinations/__init__.py +4 -3
- airbyte_cdk/destinations/destination.py +54 -20
- airbyte_cdk/destinations/vector_db_based/README.md +37 -0
- airbyte_cdk/destinations/vector_db_based/config.py +40 -17
- airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
- airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
- airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
- airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
- airbyte_cdk/entrypoint.py +153 -44
- airbyte_cdk/exception_handler.py +21 -3
- airbyte_cdk/logger.py +30 -44
- airbyte_cdk/models/__init__.py +13 -2
- airbyte_cdk/models/airbyte_protocol.py +86 -1
- airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
- airbyte_cdk/models/file_transfer_record_message.py +13 -0
- airbyte_cdk/models/well_known_types.py +1 -1
- airbyte_cdk/sources/__init__.py +5 -1
- airbyte_cdk/sources/abstract_source.py +125 -79
- airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
- airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
- airbyte_cdk/sources/config.py +3 -2
- airbyte_cdk/sources/connector_state_manager.py +49 -83
- airbyte_cdk/sources/declarative/async_job/job.py +52 -0
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
- airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
- airbyte_cdk/sources/declarative/async_job/status.py +24 -0
- airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
- airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
- airbyte_cdk/sources/declarative/auth/token.py +28 -10
- airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
- airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
- airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
- airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1213 -88
- airbyte_cdk/sources/declarative/declarative_source.py +5 -2
- airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
- airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
- airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
- airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
- airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
- airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
- airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
- airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
- airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
- airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +346 -0
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
- airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +173 -74
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
- airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
- airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
- airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
- airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
- airbyte_cdk/sources/declarative/models/__init__.py +1 -1
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1329 -595
- airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1763 -226
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
- airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
- airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
- airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
- airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
- airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
- airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
- airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
- airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
- airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
- airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
- airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
- airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
- airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
- airbyte_cdk/sources/declarative/spec/spec.py +12 -5
- airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
- airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
- airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
- airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
- airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
- airbyte_cdk/sources/declarative/types.py +19 -110
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
- airbyte_cdk/sources/embedded/base_integration.py +16 -5
- airbyte_cdk/sources/embedded/catalog.py +16 -4
- airbyte_cdk/sources/embedded/runner.py +19 -3
- airbyte_cdk/sources/embedded/tools.py +5 -2
- airbyte_cdk/sources/file_based/README.md +152 -0
- airbyte_cdk/sources/file_based/__init__.py +24 -0
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
- airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
- airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
- airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
- airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
- airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
- airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
- airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
- airbyte_cdk/sources/file_based/exceptions.py +18 -15
- airbyte_cdk/sources/file_based/file_based_source.py +140 -33
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
- airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
- airbyte_cdk/sources/file_based/remote_file.py +1 -1
- airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
- airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
- airbyte_cdk/sources/http_logger.py +8 -3
- airbyte_cdk/sources/message/__init__.py +7 -1
- airbyte_cdk/sources/message/repository.py +18 -4
- airbyte_cdk/sources/source.py +42 -38
- airbyte_cdk/sources/streams/__init__.py +2 -2
- airbyte_cdk/sources/streams/availability_strategy.py +54 -3
- airbyte_cdk/sources/streams/call_rate.py +64 -21
- airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
- airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
- airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
- airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
- airbyte_cdk/sources/streams/concurrent/README.md +7 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
- airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
- airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
- airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
- airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
- airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
- airbyte_cdk/sources/streams/core.py +412 -87
- airbyte_cdk/sources/streams/http/__init__.py +2 -1
- airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
- airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
- airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
- airbyte_cdk/sources/streams/http/exceptions.py +27 -7
- airbyte_cdk/sources/streams/http/http.py +369 -246
- airbyte_cdk/sources/streams/http/http_client.py +531 -0
- airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
- airbyte_cdk/sources/types.py +154 -0
- airbyte_cdk/sources/utils/record_helper.py +36 -21
- airbyte_cdk/sources/utils/schema_helpers.py +13 -6
- airbyte_cdk/sources/utils/slice_logger.py +4 -1
- airbyte_cdk/sources/utils/transform.py +54 -20
- airbyte_cdk/sql/_util/hashing.py +34 -0
- airbyte_cdk/sql/_util/name_normalizers.py +92 -0
- airbyte_cdk/sql/constants.py +32 -0
- airbyte_cdk/sql/exceptions.py +235 -0
- airbyte_cdk/sql/secrets.py +123 -0
- airbyte_cdk/sql/shared/__init__.py +15 -0
- airbyte_cdk/sql/shared/catalog_providers.py +145 -0
- airbyte_cdk/sql/shared/sql_processor.py +786 -0
- airbyte_cdk/sql/types.py +160 -0
- airbyte_cdk/test/catalog_builder.py +70 -18
- airbyte_cdk/test/entrypoint_wrapper.py +117 -42
- airbyte_cdk/test/mock_http/__init__.py +1 -1
- airbyte_cdk/test/mock_http/matcher.py +6 -0
- airbyte_cdk/test/mock_http/mocker.py +57 -10
- airbyte_cdk/test/mock_http/request.py +19 -3
- airbyte_cdk/test/mock_http/response.py +3 -1
- airbyte_cdk/test/mock_http/response_builder.py +32 -16
- airbyte_cdk/test/state_builder.py +18 -10
- airbyte_cdk/test/utils/__init__.py +1 -0
- airbyte_cdk/test/utils/data.py +24 -0
- airbyte_cdk/test/utils/http_mocking.py +16 -0
- airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
- airbyte_cdk/test/utils/reading.py +26 -0
- airbyte_cdk/utils/__init__.py +2 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
- airbyte_cdk/utils/analytics_message.py +10 -2
- airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
- airbyte_cdk/utils/event_timing.py +10 -10
- airbyte_cdk/utils/mapping_helpers.py +3 -1
- airbyte_cdk/utils/message_utils.py +20 -11
- airbyte_cdk/utils/print_buffer.py +75 -0
- airbyte_cdk/utils/schema_inferrer.py +198 -28
- airbyte_cdk/utils/slice_hasher.py +30 -0
- airbyte_cdk/utils/spec_schema_transformations.py +6 -3
- airbyte_cdk/utils/stream_status_utils.py +8 -1
- airbyte_cdk/utils/traced_exception.py +61 -21
- airbyte_cdk-6.17.1.dev0.dist-info/METADATA +109 -0
- airbyte_cdk-6.17.1.dev0.dist-info/RECORD +350 -0
- {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/WHEEL +1 -2
- airbyte_cdk-6.17.1.dev0.dist-info/entry_points.txt +3 -0
- airbyte_cdk/sources/declarative/create_partial.py +0 -92
- airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
- airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
- airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
- airbyte_cdk/sources/deprecated/base_source.py +0 -94
- airbyte_cdk/sources/deprecated/client.py +0 -99
- airbyte_cdk/sources/singer/__init__.py +0 -8
- airbyte_cdk/sources/singer/singer_helpers.py +0 -304
- airbyte_cdk/sources/singer/source.py +0 -186
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
- airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
- airbyte_cdk/sources/streams/http/auth/core.py +0 -29
- airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
- airbyte_cdk/sources/streams/http/auth/token.py +0 -47
- airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
- airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
- airbyte_cdk/sources/utils/schema_models.py +0 -84
- airbyte_cdk-0.72.0.dist-info/METADATA +0 -243
- airbyte_cdk-0.72.0.dist-info/RECORD +0 -466
- airbyte_cdk-0.72.0.dist-info/top_level.txt +0 -3
- source_declarative_manifest/main.py +0 -29
- unit_tests/connector_builder/__init__.py +0 -3
- unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
- unit_tests/connector_builder/test_message_grouper.py +0 -713
- unit_tests/connector_builder/utils.py +0 -27
- unit_tests/destinations/test_destination.py +0 -243
- unit_tests/singer/test_singer_helpers.py +0 -56
- unit_tests/singer/test_singer_source.py +0 -112
- unit_tests/sources/__init__.py +0 -0
- unit_tests/sources/concurrent_source/__init__.py +0 -3
- unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
- unit_tests/sources/declarative/__init__.py +0 -3
- unit_tests/sources/declarative/auth/__init__.py +0 -3
- unit_tests/sources/declarative/auth/test_oauth.py +0 -331
- unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
- unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
- unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
- unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
- unit_tests/sources/declarative/checks/__init__.py +0 -3
- unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
- unit_tests/sources/declarative/decoders/__init__.py +0 -0
- unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
- unit_tests/sources/declarative/external_component.py +0 -13
- unit_tests/sources/declarative/extractors/__init__.py +0 -3
- unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
- unit_tests/sources/declarative/incremental/__init__.py +0 -0
- unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
- unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
- unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
- unit_tests/sources/declarative/interpolation/__init__.py +0 -3
- unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
- unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
- unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
- unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
- unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
- unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
- unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
- unit_tests/sources/declarative/parsers/__init__.py +0 -3
- unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
- unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
- unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1841
- unit_tests/sources/declarative/parsers/testing_components.py +0 -36
- unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
- unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
- unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
- unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
- unit_tests/sources/declarative/requesters/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
- unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
- unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
- unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
- unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
- unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
- unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
- unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
- unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
- unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
- unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
- unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
- unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
- unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
- unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
- unit_tests/sources/declarative/retrievers/__init__.py +0 -3
- unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
- unit_tests/sources/declarative/schema/__init__.py +0 -6
- unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
- unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
- unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
- unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
- unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
- unit_tests/sources/declarative/states/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
- unit_tests/sources/declarative/test_create_partial.py +0 -83
- unit_tests/sources/declarative/test_declarative_stream.py +0 -103
- unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
- unit_tests/sources/declarative/test_types.py +0 -39
- unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
- unit_tests/sources/file_based/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
- unit_tests/sources/file_based/config/__init__.py +0 -0
- unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
- unit_tests/sources/file_based/config/test_csv_format.py +0 -34
- unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
- unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
- unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
- unit_tests/sources/file_based/file_types/__init__.py +0 -0
- unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
- unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
- unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
- unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
- unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
- unit_tests/sources/file_based/helpers.py +0 -70
- unit_tests/sources/file_based/in_memory_files_source.py +0 -211
- unit_tests/sources/file_based/scenarios/__init__.py +0 -0
- unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
- unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
- unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
- unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
- unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
- unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
- unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
- unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
- unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
- unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
- unit_tests/sources/file_based/stream/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
- unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
- unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
- unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
- unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
- unit_tests/sources/file_based/test_scenarios.py +0 -253
- unit_tests/sources/file_based/test_schema_helpers.py +0 -346
- unit_tests/sources/fixtures/__init__.py +0 -3
- unit_tests/sources/fixtures/source_test_fixture.py +0 -153
- unit_tests/sources/message/__init__.py +0 -0
- unit_tests/sources/message/test_repository.py +0 -153
- unit_tests/sources/streams/__init__.py +0 -0
- unit_tests/sources/streams/concurrent/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
- unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
- unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
- unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
- unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
- unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
- unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
- unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
- unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
- unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
- unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
- unit_tests/sources/streams/http/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/test_auth.py +0 -173
- unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
- unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
- unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
- unit_tests/sources/streams/http/test_http.py +0 -635
- unit_tests/sources/streams/test_availability_strategy.py +0 -70
- unit_tests/sources/streams/test_call_rate.py +0 -300
- unit_tests/sources/streams/test_stream_read.py +0 -405
- unit_tests/sources/streams/test_streams_core.py +0 -184
- unit_tests/sources/test_abstract_source.py +0 -1442
- unit_tests/sources/test_concurrent_source.py +0 -112
- unit_tests/sources/test_config.py +0 -92
- unit_tests/sources/test_connector_state_manager.py +0 -482
- unit_tests/sources/test_http_logger.py +0 -252
- unit_tests/sources/test_integration_source.py +0 -86
- unit_tests/sources/test_source.py +0 -684
- unit_tests/sources/test_source_read.py +0 -460
- unit_tests/test/__init__.py +0 -0
- unit_tests/test/mock_http/__init__.py +0 -0
- unit_tests/test/mock_http/test_matcher.py +0 -53
- unit_tests/test/mock_http/test_mocker.py +0 -214
- unit_tests/test/mock_http/test_request.py +0 -117
- unit_tests/test/mock_http/test_response_builder.py +0 -177
- unit_tests/test/test_entrypoint_wrapper.py +0 -240
- unit_tests/utils/__init__.py +0 -0
- unit_tests/utils/test_datetime_format_inferrer.py +0 -60
- unit_tests/utils/test_mapping_helpers.py +0 -54
- unit_tests/utils/test_message_utils.py +0 -91
- unit_tests/utils/test_rate_limiting.py +0 -26
- unit_tests/utils/test_schema_inferrer.py +0 -202
- unit_tests/utils/test_secret_utils.py +0 -135
- unit_tests/utils/test_stream_status_utils.py +0 -61
- unit_tests/utils/test_traced_exception.py +0 -107
- /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
- {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
- {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
- {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
- {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/LICENSE.txt +0 -0
@@ -4,651 +4,1108 @@
|
|
4
4
|
from __future__ import annotations
|
5
5
|
|
6
6
|
from enum import Enum
|
7
|
-
from typing import Any, Dict, List, Optional, Union
|
7
|
+
from typing import Any, Dict, List, Literal, Optional, Union
|
8
8
|
|
9
|
-
from pydantic import BaseModel, Extra, Field
|
10
|
-
from typing_extensions import Literal
|
9
|
+
from pydantic.v1 import BaseModel, Extra, Field
|
11
10
|
|
12
11
|
|
13
12
|
class AuthFlowType(Enum):
|
14
|
-
oauth2_0 =
|
15
|
-
oauth1_0 =
|
13
|
+
oauth2_0 = "oauth2.0"
|
14
|
+
oauth1_0 = "oauth1.0"
|
16
15
|
|
17
16
|
|
18
17
|
class BasicHttpAuthenticator(BaseModel):
|
19
|
-
type: Literal[
|
18
|
+
type: Literal["BasicHttpAuthenticator"]
|
20
19
|
username: str = Field(
|
21
20
|
...,
|
22
|
-
description=
|
21
|
+
description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.",
|
23
22
|
examples=["{{ config['username'] }}", "{{ config['api_key'] }}"],
|
24
|
-
title=
|
23
|
+
title="Username",
|
25
24
|
)
|
26
25
|
password: Optional[str] = Field(
|
27
|
-
|
28
|
-
description=
|
29
|
-
examples=["{{ config['password'] }}",
|
30
|
-
title=
|
26
|
+
"",
|
27
|
+
description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.",
|
28
|
+
examples=["{{ config['password'] }}", ""],
|
29
|
+
title="Password",
|
31
30
|
)
|
32
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
31
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
33
32
|
|
34
33
|
|
35
34
|
class BearerAuthenticator(BaseModel):
|
36
|
-
type: Literal[
|
35
|
+
type: Literal["BearerAuthenticator"]
|
37
36
|
api_token: str = Field(
|
38
37
|
...,
|
39
|
-
description=
|
38
|
+
description="Token to inject as request header for authenticating with the API.",
|
40
39
|
examples=["{{ config['api_key'] }}", "{{ config['token'] }}"],
|
41
|
-
title=
|
40
|
+
title="Bearer Token",
|
42
41
|
)
|
43
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
42
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
44
43
|
|
45
44
|
|
46
45
|
class CheckStream(BaseModel):
|
47
|
-
type: Literal[
|
46
|
+
type: Literal["CheckStream"]
|
48
47
|
stream_names: List[str] = Field(
|
49
48
|
...,
|
50
|
-
description=
|
51
|
-
examples=[[
|
52
|
-
title=
|
49
|
+
description="Names of the streams to try reading from when running a check operation.",
|
50
|
+
examples=[["users"], ["users", "contacts"]],
|
51
|
+
title="Stream Names",
|
53
52
|
)
|
54
53
|
|
55
54
|
|
55
|
+
class ConcurrencyLevel(BaseModel):
|
56
|
+
type: Optional[Literal["ConcurrencyLevel"]] = None
|
57
|
+
default_concurrency: Union[int, str] = Field(
|
58
|
+
...,
|
59
|
+
description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.",
|
60
|
+
examples=[10, "{{ config['num_workers'] or 10 }}"],
|
61
|
+
title="Default Concurrency",
|
62
|
+
)
|
63
|
+
max_concurrency: Optional[int] = Field(
|
64
|
+
None,
|
65
|
+
description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.",
|
66
|
+
examples=[20, 100],
|
67
|
+
title="Max Concurrency",
|
68
|
+
)
|
69
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
70
|
+
|
71
|
+
|
56
72
|
class ConstantBackoffStrategy(BaseModel):
|
57
|
-
type: Literal[
|
73
|
+
type: Literal["ConstantBackoffStrategy"]
|
58
74
|
backoff_time_in_seconds: Union[float, str] = Field(
|
59
75
|
...,
|
60
|
-
description=
|
76
|
+
description="Backoff time in seconds.",
|
61
77
|
examples=[30, 30.5, "{{ config['backoff_time'] }}"],
|
62
|
-
title=
|
78
|
+
title="Backoff Time",
|
79
|
+
)
|
80
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
81
|
+
|
82
|
+
|
83
|
+
class CursorPagination(BaseModel):
|
84
|
+
type: Literal["CursorPagination"]
|
85
|
+
cursor_value: str = Field(
|
86
|
+
...,
|
87
|
+
description="Value of the cursor defining the next page to fetch.",
|
88
|
+
examples=[
|
89
|
+
"{{ headers.link.next.cursor }}",
|
90
|
+
"{{ last_record['key'] }}",
|
91
|
+
"{{ response['nextPage'] }}",
|
92
|
+
],
|
93
|
+
title="Cursor Value",
|
94
|
+
)
|
95
|
+
page_size: Optional[int] = Field(
|
96
|
+
None,
|
97
|
+
description="The number of records to include in each pages.",
|
98
|
+
examples=[100],
|
99
|
+
title="Page Size",
|
100
|
+
)
|
101
|
+
stop_condition: Optional[str] = Field(
|
102
|
+
None,
|
103
|
+
description="Template string evaluating when to stop paginating.",
|
104
|
+
examples=[
|
105
|
+
"{{ response.data.has_more is false }}",
|
106
|
+
"{{ 'next' not in headers['link'] }}",
|
107
|
+
],
|
108
|
+
title="Stop Condition",
|
63
109
|
)
|
64
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
110
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
65
111
|
|
66
112
|
|
67
113
|
class CustomAuthenticator(BaseModel):
|
68
114
|
class Config:
|
69
115
|
extra = Extra.allow
|
70
116
|
|
71
|
-
type: Literal[
|
117
|
+
type: Literal["CustomAuthenticator"]
|
72
118
|
class_name: str = Field(
|
73
119
|
...,
|
74
|
-
description=
|
75
|
-
examples=[
|
76
|
-
title=
|
120
|
+
description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.",
|
121
|
+
examples=["source_railz.components.ShortLivedTokenAuthenticator"],
|
122
|
+
title="Class Name",
|
77
123
|
)
|
78
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
124
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
79
125
|
|
80
126
|
|
81
127
|
class CustomBackoffStrategy(BaseModel):
|
82
128
|
class Config:
|
83
129
|
extra = Extra.allow
|
84
130
|
|
85
|
-
type: Literal[
|
131
|
+
type: Literal["CustomBackoffStrategy"]
|
86
132
|
class_name: str = Field(
|
87
133
|
...,
|
88
|
-
description=
|
89
|
-
examples=[
|
90
|
-
title=
|
134
|
+
description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.",
|
135
|
+
examples=["source_railz.components.MyCustomBackoffStrategy"],
|
136
|
+
title="Class Name",
|
91
137
|
)
|
92
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
138
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
93
139
|
|
94
140
|
|
95
141
|
class CustomErrorHandler(BaseModel):
|
96
142
|
class Config:
|
97
143
|
extra = Extra.allow
|
98
144
|
|
99
|
-
type: Literal[
|
145
|
+
type: Literal["CustomErrorHandler"]
|
100
146
|
class_name: str = Field(
|
101
147
|
...,
|
102
|
-
description=
|
103
|
-
examples=[
|
104
|
-
title=
|
148
|
+
description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.",
|
149
|
+
examples=["source_railz.components.MyCustomErrorHandler"],
|
150
|
+
title="Class Name",
|
105
151
|
)
|
106
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
152
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
107
153
|
|
108
154
|
|
109
155
|
class CustomIncrementalSync(BaseModel):
|
110
156
|
class Config:
|
111
157
|
extra = Extra.allow
|
112
158
|
|
113
|
-
type: Literal[
|
159
|
+
type: Literal["CustomIncrementalSync"]
|
114
160
|
class_name: str = Field(
|
115
161
|
...,
|
116
|
-
description=
|
117
|
-
examples=[
|
118
|
-
title=
|
162
|
+
description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.",
|
163
|
+
examples=["source_railz.components.MyCustomIncrementalSync"],
|
164
|
+
title="Class Name",
|
119
165
|
)
|
120
166
|
cursor_field: str = Field(
|
121
167
|
...,
|
122
|
-
description=
|
168
|
+
description="The location of the value on a record that will be used as a bookmark during sync.",
|
123
169
|
)
|
124
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
170
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
125
171
|
|
126
172
|
|
127
173
|
class CustomPaginationStrategy(BaseModel):
|
128
174
|
class Config:
|
129
175
|
extra = Extra.allow
|
130
176
|
|
131
|
-
type: Literal[
|
177
|
+
type: Literal["CustomPaginationStrategy"]
|
132
178
|
class_name: str = Field(
|
133
179
|
...,
|
134
|
-
description=
|
135
|
-
examples=[
|
136
|
-
title=
|
180
|
+
description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.",
|
181
|
+
examples=["source_railz.components.MyCustomPaginationStrategy"],
|
182
|
+
title="Class Name",
|
137
183
|
)
|
138
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
184
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
139
185
|
|
140
186
|
|
141
187
|
class CustomRecordExtractor(BaseModel):
|
142
188
|
class Config:
|
143
189
|
extra = Extra.allow
|
144
190
|
|
145
|
-
type: Literal[
|
191
|
+
type: Literal["CustomRecordExtractor"]
|
146
192
|
class_name: str = Field(
|
147
193
|
...,
|
148
|
-
description=
|
149
|
-
examples=[
|
150
|
-
title=
|
194
|
+
description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.",
|
195
|
+
examples=["source_railz.components.MyCustomRecordExtractor"],
|
196
|
+
title="Class Name",
|
151
197
|
)
|
152
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
198
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
153
199
|
|
154
200
|
|
155
201
|
class CustomRecordFilter(BaseModel):
|
156
202
|
class Config:
|
157
203
|
extra = Extra.allow
|
158
204
|
|
159
|
-
type: Literal[
|
205
|
+
type: Literal["CustomRecordFilter"]
|
160
206
|
class_name: str = Field(
|
161
207
|
...,
|
162
|
-
description=
|
163
|
-
examples=[
|
164
|
-
title=
|
208
|
+
description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.",
|
209
|
+
examples=["source_railz.components.MyCustomCustomRecordFilter"],
|
210
|
+
title="Class Name",
|
165
211
|
)
|
166
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
212
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
167
213
|
|
168
214
|
|
169
215
|
class CustomRequester(BaseModel):
|
170
216
|
class Config:
|
171
217
|
extra = Extra.allow
|
172
218
|
|
173
|
-
type: Literal[
|
219
|
+
type: Literal["CustomRequester"]
|
174
220
|
class_name: str = Field(
|
175
221
|
...,
|
176
|
-
description=
|
177
|
-
examples=[
|
178
|
-
title=
|
222
|
+
description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.",
|
223
|
+
examples=["source_railz.components.MyCustomRecordExtractor"],
|
224
|
+
title="Class Name",
|
179
225
|
)
|
180
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
226
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
181
227
|
|
182
228
|
|
183
229
|
class CustomRetriever(BaseModel):
|
184
230
|
class Config:
|
185
231
|
extra = Extra.allow
|
186
232
|
|
187
|
-
type: Literal[
|
233
|
+
type: Literal["CustomRetriever"]
|
188
234
|
class_name: str = Field(
|
189
235
|
...,
|
190
|
-
description=
|
191
|
-
examples=[
|
192
|
-
title=
|
236
|
+
description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.",
|
237
|
+
examples=["source_railz.components.MyCustomRetriever"],
|
238
|
+
title="Class Name",
|
193
239
|
)
|
194
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
240
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
195
241
|
|
196
242
|
|
197
243
|
class CustomPartitionRouter(BaseModel):
|
198
244
|
class Config:
|
199
245
|
extra = Extra.allow
|
200
246
|
|
201
|
-
type: Literal[
|
247
|
+
type: Literal["CustomPartitionRouter"]
|
202
248
|
class_name: str = Field(
|
203
249
|
...,
|
204
|
-
description=
|
205
|
-
examples=[
|
206
|
-
title=
|
250
|
+
description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.",
|
251
|
+
examples=["source_railz.components.MyCustomPartitionRouter"],
|
252
|
+
title="Class Name",
|
207
253
|
)
|
208
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
254
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
209
255
|
|
210
256
|
|
211
257
|
class CustomSchemaLoader(BaseModel):
|
212
258
|
class Config:
|
213
259
|
extra = Extra.allow
|
214
260
|
|
215
|
-
type: Literal[
|
261
|
+
type: Literal["CustomSchemaLoader"]
|
216
262
|
class_name: str = Field(
|
217
263
|
...,
|
218
|
-
description=
|
219
|
-
examples=[
|
220
|
-
title=
|
264
|
+
description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.",
|
265
|
+
examples=["source_railz.components.MyCustomSchemaLoader"],
|
266
|
+
title="Class Name",
|
221
267
|
)
|
222
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
268
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
269
|
+
|
270
|
+
|
271
|
+
class CustomSchemaNormalization(BaseModel):
|
272
|
+
class Config:
|
273
|
+
extra = Extra.allow
|
274
|
+
|
275
|
+
type: Literal["CustomSchemaNormalization"]
|
276
|
+
class_name: str = Field(
|
277
|
+
...,
|
278
|
+
description="Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.",
|
279
|
+
examples=[
|
280
|
+
"source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer"
|
281
|
+
],
|
282
|
+
title="Class Name",
|
283
|
+
)
|
284
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
285
|
+
|
286
|
+
|
287
|
+
class CustomStateMigration(BaseModel):
|
288
|
+
class Config:
|
289
|
+
extra = Extra.allow
|
290
|
+
|
291
|
+
type: Literal["CustomStateMigration"]
|
292
|
+
class_name: str = Field(
|
293
|
+
...,
|
294
|
+
description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.",
|
295
|
+
examples=["source_railz.components.MyCustomStateMigration"],
|
296
|
+
title="Class Name",
|
297
|
+
)
|
298
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
223
299
|
|
224
300
|
|
225
301
|
class CustomTransformation(BaseModel):
|
226
302
|
class Config:
|
227
303
|
extra = Extra.allow
|
228
304
|
|
229
|
-
type: Literal[
|
305
|
+
type: Literal["CustomTransformation"]
|
230
306
|
class_name: str = Field(
|
231
307
|
...,
|
232
|
-
description=
|
233
|
-
examples=[
|
234
|
-
title=
|
308
|
+
description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.",
|
309
|
+
examples=["source_railz.components.MyCustomTransformation"],
|
310
|
+
title="Class Name",
|
235
311
|
)
|
236
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
312
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
313
|
+
|
314
|
+
|
315
|
+
class LegacyToPerPartitionStateMigration(BaseModel):
|
316
|
+
class Config:
|
317
|
+
extra = Extra.allow
|
318
|
+
|
319
|
+
type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None
|
320
|
+
|
321
|
+
|
322
|
+
class Algorithm(Enum):
|
323
|
+
HS256 = "HS256"
|
324
|
+
HS384 = "HS384"
|
325
|
+
HS512 = "HS512"
|
326
|
+
ES256 = "ES256"
|
327
|
+
ES256K = "ES256K"
|
328
|
+
ES384 = "ES384"
|
329
|
+
ES512 = "ES512"
|
330
|
+
RS256 = "RS256"
|
331
|
+
RS384 = "RS384"
|
332
|
+
RS512 = "RS512"
|
333
|
+
PS256 = "PS256"
|
334
|
+
PS384 = "PS384"
|
335
|
+
PS512 = "PS512"
|
336
|
+
EdDSA = "EdDSA"
|
337
|
+
|
338
|
+
|
339
|
+
class JwtHeaders(BaseModel):
|
340
|
+
class Config:
|
341
|
+
extra = Extra.forbid
|
342
|
+
|
343
|
+
kid: Optional[str] = Field(
|
344
|
+
None,
|
345
|
+
description="Private key ID for user account.",
|
346
|
+
examples=["{{ config['kid'] }}"],
|
347
|
+
title="Key Identifier",
|
348
|
+
)
|
349
|
+
typ: Optional[str] = Field(
|
350
|
+
"JWT",
|
351
|
+
description="The media type of the complete JWT.",
|
352
|
+
examples=["JWT"],
|
353
|
+
title="Type",
|
354
|
+
)
|
355
|
+
cty: Optional[str] = Field(
|
356
|
+
None,
|
357
|
+
description="Content type of JWT header.",
|
358
|
+
examples=["JWT"],
|
359
|
+
title="Content Type",
|
360
|
+
)
|
361
|
+
|
362
|
+
|
363
|
+
class JwtPayload(BaseModel):
|
364
|
+
class Config:
|
365
|
+
extra = Extra.forbid
|
366
|
+
|
367
|
+
iss: Optional[str] = Field(
|
368
|
+
None,
|
369
|
+
description="The user/principal that issued the JWT. Commonly a value unique to the user.",
|
370
|
+
examples=["{{ config['iss'] }}"],
|
371
|
+
title="Issuer",
|
372
|
+
)
|
373
|
+
sub: Optional[str] = Field(
|
374
|
+
None,
|
375
|
+
description="The subject of the JWT. Commonly defined by the API.",
|
376
|
+
title="Subject",
|
377
|
+
)
|
378
|
+
aud: Optional[str] = Field(
|
379
|
+
None,
|
380
|
+
description="The recipient that the JWT is intended for. Commonly defined by the API.",
|
381
|
+
examples=["appstoreconnect-v1"],
|
382
|
+
title="Audience",
|
383
|
+
)
|
384
|
+
|
385
|
+
|
386
|
+
class JwtAuthenticator(BaseModel):
|
387
|
+
type: Literal["JwtAuthenticator"]
|
388
|
+
secret_key: str = Field(
|
389
|
+
...,
|
390
|
+
description="Secret used to sign the JSON web token.",
|
391
|
+
examples=["{{ config['secret_key'] }}"],
|
392
|
+
)
|
393
|
+
base64_encode_secret_key: Optional[bool] = Field(
|
394
|
+
False,
|
395
|
+
description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.',
|
396
|
+
)
|
397
|
+
algorithm: Algorithm = Field(
|
398
|
+
...,
|
399
|
+
description="Algorithm used to sign the JSON web token.",
|
400
|
+
examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"],
|
401
|
+
)
|
402
|
+
token_duration: Optional[int] = Field(
|
403
|
+
1200,
|
404
|
+
description="The amount of time in seconds a JWT token can be valid after being issued.",
|
405
|
+
examples=[1200, 3600],
|
406
|
+
title="Token Duration",
|
407
|
+
)
|
408
|
+
header_prefix: Optional[str] = Field(
|
409
|
+
None,
|
410
|
+
description="The prefix to be used within the Authentication header.",
|
411
|
+
examples=["Bearer", "Basic"],
|
412
|
+
title="Header Prefix",
|
413
|
+
)
|
414
|
+
jwt_headers: Optional[JwtHeaders] = Field(
|
415
|
+
None,
|
416
|
+
description="JWT headers used when signing JSON web token.",
|
417
|
+
title="JWT Headers",
|
418
|
+
)
|
419
|
+
additional_jwt_headers: Optional[Dict[str, Any]] = Field(
|
420
|
+
None,
|
421
|
+
description="Additional headers to be included with the JWT headers object.",
|
422
|
+
title="Additional JWT Headers",
|
423
|
+
)
|
424
|
+
jwt_payload: Optional[JwtPayload] = Field(
|
425
|
+
None,
|
426
|
+
description="JWT Payload used when signing JSON web token.",
|
427
|
+
title="JWT Payload",
|
428
|
+
)
|
429
|
+
additional_jwt_payload: Optional[Dict[str, Any]] = Field(
|
430
|
+
None,
|
431
|
+
description="Additional properties to be added to the JWT payload.",
|
432
|
+
title="Additional JWT Payload Properties",
|
433
|
+
)
|
434
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
237
435
|
|
238
436
|
|
239
437
|
class RefreshTokenUpdater(BaseModel):
|
240
438
|
refresh_token_name: Optional[str] = Field(
|
241
|
-
|
242
|
-
description=
|
243
|
-
examples=[
|
244
|
-
title=
|
439
|
+
"refresh_token",
|
440
|
+
description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.",
|
441
|
+
examples=["refresh_token"],
|
442
|
+
title="Refresh Token Property Name",
|
245
443
|
)
|
246
444
|
access_token_config_path: Optional[List[str]] = Field(
|
247
|
-
[
|
248
|
-
description=
|
249
|
-
examples=[[
|
250
|
-
title=
|
445
|
+
["credentials", "access_token"],
|
446
|
+
description="Config path to the access token. Make sure the field actually exists in the config.",
|
447
|
+
examples=[["credentials", "access_token"], ["access_token"]],
|
448
|
+
title="Config Path To Access Token",
|
251
449
|
)
|
252
450
|
refresh_token_config_path: Optional[List[str]] = Field(
|
253
|
-
[
|
254
|
-
description=
|
255
|
-
examples=[[
|
256
|
-
title=
|
451
|
+
["credentials", "refresh_token"],
|
452
|
+
description="Config path to the access token. Make sure the field actually exists in the config.",
|
453
|
+
examples=[["credentials", "refresh_token"], ["refresh_token"]],
|
454
|
+
title="Config Path To Refresh Token",
|
257
455
|
)
|
258
456
|
token_expiry_date_config_path: Optional[List[str]] = Field(
|
259
|
-
[
|
260
|
-
description=
|
261
|
-
examples=[[
|
262
|
-
title=
|
457
|
+
["credentials", "token_expiry_date"],
|
458
|
+
description="Config path to the expiry date. Make sure actually exists in the config.",
|
459
|
+
examples=[["credentials", "token_expiry_date"]],
|
460
|
+
title="Config Path To Expiry Date",
|
461
|
+
)
|
462
|
+
refresh_token_error_status_codes: Optional[List[int]] = Field(
|
463
|
+
[],
|
464
|
+
description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error",
|
465
|
+
examples=[[400, 500]],
|
466
|
+
title="Refresh Token Error Status Codes",
|
467
|
+
)
|
468
|
+
refresh_token_error_key: Optional[str] = Field(
|
469
|
+
"",
|
470
|
+
description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).",
|
471
|
+
examples=["error"],
|
472
|
+
title="Refresh Token Error Key",
|
473
|
+
)
|
474
|
+
refresh_token_error_values: Optional[List[str]] = Field(
|
475
|
+
[],
|
476
|
+
description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error',
|
477
|
+
examples=[["invalid_grant", "invalid_permissions"]],
|
478
|
+
title="Refresh Token Error Values",
|
263
479
|
)
|
264
480
|
|
265
481
|
|
266
482
|
class OAuthAuthenticator(BaseModel):
|
267
|
-
type: Literal[
|
483
|
+
type: Literal["OAuthAuthenticator"]
|
268
484
|
client_id: str = Field(
|
269
485
|
...,
|
270
|
-
description=
|
486
|
+
description="The OAuth client ID. Fill it in the user inputs.",
|
271
487
|
examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"],
|
272
|
-
title=
|
488
|
+
title="Client ID",
|
273
489
|
)
|
274
490
|
client_secret: str = Field(
|
275
491
|
...,
|
276
|
-
description=
|
492
|
+
description="The OAuth client secret. Fill it in the user inputs.",
|
277
493
|
examples=[
|
278
494
|
"{{ config['client_secret }}",
|
279
495
|
"{{ config['credentials']['client_secret }}",
|
280
496
|
],
|
281
|
-
title=
|
497
|
+
title="Client Secret",
|
282
498
|
)
|
283
499
|
refresh_token: Optional[str] = Field(
|
284
500
|
None,
|
285
|
-
description=
|
501
|
+
description="Credential artifact used to get a new access token.",
|
286
502
|
examples=[
|
287
503
|
"{{ config['refresh_token'] }}",
|
288
504
|
"{{ config['credentials]['refresh_token'] }}",
|
289
505
|
],
|
290
|
-
title=
|
506
|
+
title="Refresh Token",
|
291
507
|
)
|
292
|
-
token_refresh_endpoint: str = Field(
|
293
|
-
|
294
|
-
description=
|
295
|
-
examples=[
|
296
|
-
title=
|
508
|
+
token_refresh_endpoint: Optional[str] = Field(
|
509
|
+
None,
|
510
|
+
description="The full URL to call to obtain a new access token.",
|
511
|
+
examples=["https://connect.squareup.com/oauth2/token"],
|
512
|
+
title="Token Refresh Endpoint",
|
297
513
|
)
|
298
514
|
access_token_name: Optional[str] = Field(
|
299
|
-
|
300
|
-
description=
|
301
|
-
examples=[
|
302
|
-
title=
|
515
|
+
"access_token",
|
516
|
+
description="The name of the property which contains the access token in the response from the token refresh endpoint.",
|
517
|
+
examples=["access_token"],
|
518
|
+
title="Access Token Property Name",
|
519
|
+
)
|
520
|
+
access_token_value: Optional[str] = Field(
|
521
|
+
None,
|
522
|
+
description="The value of the access_token to bypass the token refreshing using `refresh_token`.",
|
523
|
+
examples=["secret_access_token_value"],
|
524
|
+
title="Access Token Value",
|
303
525
|
)
|
304
526
|
expires_in_name: Optional[str] = Field(
|
305
|
-
|
306
|
-
description=
|
307
|
-
examples=[
|
308
|
-
title=
|
527
|
+
"expires_in",
|
528
|
+
description="The name of the property which contains the expiry date in the response from the token refresh endpoint.",
|
529
|
+
examples=["expires_in"],
|
530
|
+
title="Token Expiry Property Name",
|
309
531
|
)
|
310
532
|
grant_type: Optional[str] = Field(
|
311
|
-
|
312
|
-
description=
|
313
|
-
examples=[
|
314
|
-
title=
|
533
|
+
"refresh_token",
|
534
|
+
description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
|
535
|
+
examples=["refresh_token", "client_credentials"],
|
536
|
+
title="Grant Type",
|
315
537
|
)
|
316
538
|
refresh_request_body: Optional[Dict[str, Any]] = Field(
|
317
539
|
None,
|
318
|
-
description=
|
540
|
+
description="Body of the request sent to get a new access token.",
|
319
541
|
examples=[
|
320
542
|
{
|
321
|
-
|
322
|
-
|
323
|
-
|
543
|
+
"applicationId": "{{ config['application_id'] }}",
|
544
|
+
"applicationSecret": "{{ config['application_secret'] }}",
|
545
|
+
"token": "{{ config['token'] }}",
|
324
546
|
}
|
325
547
|
],
|
326
|
-
title=
|
548
|
+
title="Refresh Request Body",
|
327
549
|
)
|
328
550
|
scopes: Optional[List[str]] = Field(
|
329
551
|
None,
|
330
|
-
description=
|
331
|
-
examples=[
|
332
|
-
|
333
|
-
],
|
334
|
-
title='Scopes',
|
552
|
+
description="List of scopes that should be granted to the access token.",
|
553
|
+
examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]],
|
554
|
+
title="Scopes",
|
335
555
|
)
|
336
556
|
token_expiry_date: Optional[str] = Field(
|
337
557
|
None,
|
338
|
-
description=
|
339
|
-
examples=[
|
340
|
-
title=
|
558
|
+
description="The access token expiry date.",
|
559
|
+
examples=["2023-04-06T07:12:10.421833+00:00", 1680842386],
|
560
|
+
title="Token Expiry Date",
|
341
561
|
)
|
342
562
|
token_expiry_date_format: Optional[str] = Field(
|
343
563
|
None,
|
344
|
-
description=
|
345
|
-
examples=[
|
346
|
-
title=
|
564
|
+
description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.",
|
565
|
+
examples=["%Y-%m-%d %H:%M:%S.%f+00:00"],
|
566
|
+
title="Token Expiry Date Format",
|
347
567
|
)
|
348
568
|
refresh_token_updater: Optional[RefreshTokenUpdater] = Field(
|
349
569
|
None,
|
350
|
-
description=
|
351
|
-
title=
|
570
|
+
description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.",
|
571
|
+
title="Token Updater",
|
352
572
|
)
|
353
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
573
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
574
|
+
|
575
|
+
|
576
|
+
class DpathExtractor(BaseModel):
|
577
|
+
type: Literal["DpathExtractor"]
|
578
|
+
field_path: List[str] = Field(
|
579
|
+
...,
|
580
|
+
description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
|
581
|
+
examples=[
|
582
|
+
["data"],
|
583
|
+
["data", "records"],
|
584
|
+
["data", "{{ parameters.name }}"],
|
585
|
+
["data", "*", "record"],
|
586
|
+
],
|
587
|
+
title="Field Path",
|
588
|
+
)
|
589
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
590
|
+
|
591
|
+
|
592
|
+
class ResponseToFileExtractor(BaseModel):
|
593
|
+
type: Literal["ResponseToFileExtractor"]
|
594
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
354
595
|
|
355
596
|
|
356
597
|
class ExponentialBackoffStrategy(BaseModel):
|
357
|
-
type: Literal[
|
598
|
+
type: Literal["ExponentialBackoffStrategy"]
|
358
599
|
factor: Optional[Union[float, str]] = Field(
|
359
600
|
5,
|
360
|
-
description=
|
361
|
-
examples=[5, 5.5,
|
362
|
-
title=
|
601
|
+
description="Multiplicative constant applied on each retry.",
|
602
|
+
examples=[5, 5.5, "10"],
|
603
|
+
title="Factor",
|
363
604
|
)
|
364
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
605
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
365
606
|
|
366
607
|
|
367
608
|
class SessionTokenRequestBearerAuthenticator(BaseModel):
|
368
|
-
type: Literal[
|
609
|
+
type: Literal["Bearer"]
|
369
610
|
|
370
611
|
|
371
612
|
class HttpMethod(Enum):
|
372
|
-
GET =
|
373
|
-
POST =
|
613
|
+
GET = "GET"
|
614
|
+
POST = "POST"
|
374
615
|
|
375
616
|
|
376
617
|
class Action(Enum):
|
377
|
-
SUCCESS =
|
378
|
-
FAIL =
|
379
|
-
RETRY =
|
380
|
-
IGNORE =
|
618
|
+
SUCCESS = "SUCCESS"
|
619
|
+
FAIL = "FAIL"
|
620
|
+
RETRY = "RETRY"
|
621
|
+
IGNORE = "IGNORE"
|
622
|
+
RATE_LIMITED = "RATE_LIMITED"
|
623
|
+
|
624
|
+
|
625
|
+
class FailureType(Enum):
|
626
|
+
system_error = "system_error"
|
627
|
+
config_error = "config_error"
|
628
|
+
transient_error = "transient_error"
|
381
629
|
|
382
630
|
|
383
631
|
class HttpResponseFilter(BaseModel):
|
384
|
-
type: Literal[
|
385
|
-
action: Action = Field(
|
386
|
-
|
387
|
-
description=
|
388
|
-
examples=[
|
389
|
-
title=
|
632
|
+
type: Literal["HttpResponseFilter"]
|
633
|
+
action: Optional[Action] = Field(
|
634
|
+
None,
|
635
|
+
description="Action to execute if a response matches the filter.",
|
636
|
+
examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"],
|
637
|
+
title="Action",
|
638
|
+
)
|
639
|
+
failure_type: Optional[FailureType] = Field(
|
640
|
+
None,
|
641
|
+
description="Failure type of traced exception if a response matches the filter.",
|
642
|
+
examples=["system_error", "config_error", "transient_error"],
|
643
|
+
title="Failure Type",
|
390
644
|
)
|
391
645
|
error_message: Optional[str] = Field(
|
392
646
|
None,
|
393
|
-
description=
|
394
|
-
title=
|
647
|
+
description="Error Message to display if the response matches the filter.",
|
648
|
+
title="Error Message",
|
395
649
|
)
|
396
650
|
error_message_contains: Optional[str] = Field(
|
397
651
|
None,
|
398
|
-
description=
|
399
|
-
example=[
|
400
|
-
title=
|
652
|
+
description="Match the response if its error message contains the substring.",
|
653
|
+
example=["This API operation is not enabled for this site"],
|
654
|
+
title="Error Message Substring",
|
401
655
|
)
|
402
656
|
http_codes: Optional[List[int]] = Field(
|
403
657
|
None,
|
404
|
-
description=
|
658
|
+
description="Match the response if its HTTP code is included in this list.",
|
405
659
|
examples=[[420, 429], [500]],
|
406
|
-
title=
|
660
|
+
title="HTTP Codes",
|
661
|
+
unique_items=True,
|
407
662
|
)
|
408
663
|
predicate: Optional[str] = Field(
|
409
664
|
None,
|
410
|
-
description=
|
665
|
+
description="Match the response if the predicate evaluates to true.",
|
411
666
|
examples=[
|
412
667
|
"{{ 'Too much requests' in response }}",
|
413
668
|
"{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}",
|
414
669
|
],
|
415
|
-
title=
|
670
|
+
title="Predicate",
|
416
671
|
)
|
417
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
672
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
673
|
+
|
674
|
+
|
675
|
+
class TypesMap(BaseModel):
|
676
|
+
target_type: Union[str, List[str]]
|
677
|
+
current_type: Union[str, List[str]]
|
678
|
+
|
679
|
+
|
680
|
+
class SchemaTypeIdentifier(BaseModel):
|
681
|
+
type: Optional[Literal["SchemaTypeIdentifier"]] = None
|
682
|
+
schema_pointer: Optional[List[str]] = Field(
|
683
|
+
[],
|
684
|
+
description="List of nested fields defining the schema field path to extract. Defaults to [].",
|
685
|
+
title="Schema Path",
|
686
|
+
)
|
687
|
+
key_pointer: List[str] = Field(
|
688
|
+
...,
|
689
|
+
description="List of potentially nested fields describing the full path of the field key to extract.",
|
690
|
+
title="Key Path",
|
691
|
+
)
|
692
|
+
type_pointer: Optional[List[str]] = Field(
|
693
|
+
None,
|
694
|
+
description="List of potentially nested fields describing the full path of the field type to extract.",
|
695
|
+
title="Type Path",
|
696
|
+
)
|
697
|
+
types_mapping: Optional[List[TypesMap]] = None
|
698
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
418
699
|
|
419
700
|
|
420
701
|
class InlineSchemaLoader(BaseModel):
|
421
|
-
type: Literal[
|
702
|
+
type: Literal["InlineSchemaLoader"]
|
422
703
|
schema_: Optional[Dict[str, Any]] = Field(
|
423
704
|
None,
|
424
|
-
alias=
|
705
|
+
alias="schema",
|
425
706
|
description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.',
|
426
|
-
title=
|
707
|
+
title="Schema",
|
427
708
|
)
|
428
709
|
|
429
710
|
|
430
711
|
class JsonFileSchemaLoader(BaseModel):
|
431
|
-
type: Literal[
|
712
|
+
type: Literal["JsonFileSchemaLoader"]
|
432
713
|
file_path: Optional[str] = Field(
|
433
714
|
None,
|
434
715
|
description="Path to the JSON file defining the schema. The path is relative to the connector module's root.",
|
435
|
-
example=[
|
436
|
-
title=
|
716
|
+
example=["./schemas/users.json"],
|
717
|
+
title="File Path",
|
437
718
|
)
|
438
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
719
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
439
720
|
|
440
721
|
|
441
722
|
class JsonDecoder(BaseModel):
|
442
|
-
type: Literal[
|
723
|
+
type: Literal["JsonDecoder"]
|
724
|
+
|
725
|
+
|
726
|
+
class JsonlDecoder(BaseModel):
|
727
|
+
type: Literal["JsonlDecoder"]
|
728
|
+
|
729
|
+
|
730
|
+
class KeysToLower(BaseModel):
|
731
|
+
type: Literal["KeysToLower"]
|
732
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
733
|
+
|
734
|
+
|
735
|
+
class KeysToSnakeCase(BaseModel):
|
736
|
+
type: Literal["KeysToSnakeCase"]
|
737
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
738
|
+
|
739
|
+
|
740
|
+
class KeysReplace(BaseModel):
|
741
|
+
type: Literal["KeysReplace"]
|
742
|
+
old: str = Field(
|
743
|
+
...,
|
744
|
+
description="Old value to replace.",
|
745
|
+
examples=[" ", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
|
746
|
+
title="Old value",
|
747
|
+
)
|
748
|
+
new: str = Field(
|
749
|
+
...,
|
750
|
+
description="New value to set.",
|
751
|
+
examples=["_", "{{ record.id }}", "{{ config['id'] }}", "{{ stream_slice['id'] }}"],
|
752
|
+
title="New value",
|
753
|
+
)
|
754
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
755
|
+
|
756
|
+
|
757
|
+
class FlattenFields(BaseModel):
|
758
|
+
type: Literal["FlattenFields"]
|
759
|
+
flatten_lists: Optional[bool] = Field(
|
760
|
+
True,
|
761
|
+
description="Whether to flatten lists or leave it as is. Default is True.",
|
762
|
+
title="Flatten Lists",
|
763
|
+
)
|
764
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
765
|
+
|
766
|
+
|
767
|
+
class IterableDecoder(BaseModel):
|
768
|
+
type: Literal["IterableDecoder"]
|
769
|
+
|
770
|
+
|
771
|
+
class XmlDecoder(BaseModel):
|
772
|
+
type: Literal["XmlDecoder"]
|
773
|
+
|
774
|
+
|
775
|
+
class CustomDecoder(BaseModel):
|
776
|
+
class Config:
|
777
|
+
extra = Extra.allow
|
778
|
+
|
779
|
+
type: Literal["CustomDecoder"]
|
780
|
+
class_name: str = Field(
|
781
|
+
...,
|
782
|
+
description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.",
|
783
|
+
examples=["source_amazon_ads.components.GzipJsonlDecoder"],
|
784
|
+
title="Class Name",
|
785
|
+
)
|
786
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
787
|
+
|
788
|
+
|
789
|
+
class GzipJsonDecoder(BaseModel):
|
790
|
+
class Config:
|
791
|
+
extra = Extra.allow
|
792
|
+
|
793
|
+
type: Literal["GzipJsonDecoder"]
|
794
|
+
encoding: Optional[str] = "utf-8"
|
795
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
443
796
|
|
444
797
|
|
445
798
|
class MinMaxDatetime(BaseModel):
|
446
|
-
type: Literal[
|
799
|
+
type: Literal["MinMaxDatetime"]
|
447
800
|
datetime: str = Field(
|
448
801
|
...,
|
449
|
-
description=
|
450
|
-
examples=[
|
451
|
-
title=
|
802
|
+
description="Datetime value.",
|
803
|
+
examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"],
|
804
|
+
title="Datetime",
|
452
805
|
)
|
453
806
|
datetime_format: Optional[str] = Field(
|
454
|
-
|
455
|
-
description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
|
456
|
-
examples=[
|
457
|
-
title=
|
807
|
+
"",
|
808
|
+
description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
|
809
|
+
examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
|
810
|
+
title="Datetime Format",
|
458
811
|
)
|
459
812
|
max_datetime: Optional[str] = Field(
|
460
813
|
None,
|
461
|
-
description=
|
462
|
-
examples=[
|
463
|
-
title=
|
814
|
+
description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.",
|
815
|
+
examples=["2021-01-01T00:00:00Z", "2021-01-01"],
|
816
|
+
title="Max Datetime",
|
464
817
|
)
|
465
818
|
min_datetime: Optional[str] = Field(
|
466
819
|
None,
|
467
|
-
description=
|
468
|
-
examples=[
|
469
|
-
title=
|
820
|
+
description="Floor applied on the datetime value. Must be formatted with the datetime_format field.",
|
821
|
+
examples=["2010-01-01T00:00:00Z", "2010-01-01"],
|
822
|
+
title="Min Datetime",
|
470
823
|
)
|
471
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
824
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
472
825
|
|
473
826
|
|
474
827
|
class NoAuth(BaseModel):
|
475
|
-
type: Literal[
|
476
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
828
|
+
type: Literal["NoAuth"]
|
829
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
477
830
|
|
478
831
|
|
479
832
|
class NoPagination(BaseModel):
|
480
|
-
type: Literal[
|
833
|
+
type: Literal["NoPagination"]
|
834
|
+
|
835
|
+
|
836
|
+
class State(BaseModel):
|
837
|
+
class Config:
|
838
|
+
extra = Extra.allow
|
839
|
+
|
840
|
+
min: int
|
841
|
+
max: int
|
842
|
+
|
843
|
+
|
844
|
+
class OauthConnectorInputSpecification(BaseModel):
|
845
|
+
class Config:
|
846
|
+
extra = Extra.allow
|
847
|
+
|
848
|
+
consent_url: str = Field(
|
849
|
+
...,
|
850
|
+
description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
|
851
|
+
examples=[
|
852
|
+
"https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
|
853
|
+
"https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}",
|
854
|
+
],
|
855
|
+
title="Consent URL",
|
856
|
+
)
|
857
|
+
scope: Optional[str] = Field(
|
858
|
+
None,
|
859
|
+
description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
|
860
|
+
examples=["user:read user:read_orders workspaces:read"],
|
861
|
+
title="Scopes",
|
862
|
+
)
|
863
|
+
access_token_url: str = Field(
|
864
|
+
...,
|
865
|
+
description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
|
866
|
+
examples=[
|
867
|
+
"https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
|
868
|
+
],
|
869
|
+
title="Access Token URL",
|
870
|
+
)
|
871
|
+
access_token_headers: Optional[Dict[str, Any]] = Field(
|
872
|
+
None,
|
873
|
+
description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
|
874
|
+
examples=[{"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"}],
|
875
|
+
title="Access Token Headers",
|
876
|
+
)
|
877
|
+
access_token_params: Optional[Dict[str, Any]] = Field(
|
878
|
+
None,
|
879
|
+
description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
|
880
|
+
examples=[
|
881
|
+
{
|
882
|
+
"{auth_code_key}": "{{auth_code_key}}",
|
883
|
+
"{client_id_key}": "{{client_id_key}}",
|
884
|
+
"{client_secret_key}": "{{client_secret_key}}",
|
885
|
+
}
|
886
|
+
],
|
887
|
+
title="Access Token Query Params (Json Encoded)",
|
888
|
+
)
|
889
|
+
extract_output: List[str] = Field(
|
890
|
+
...,
|
891
|
+
description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.",
|
892
|
+
examples=[["access_token", "refresh_token", "other_field"]],
|
893
|
+
title="Extract Output",
|
894
|
+
)
|
895
|
+
state: Optional[State] = Field(
|
896
|
+
None,
|
897
|
+
description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity.",
|
898
|
+
examples=[{"min": 7, "max": 128}],
|
899
|
+
title="Configurable State Query Param",
|
900
|
+
)
|
901
|
+
client_id_key: Optional[str] = Field(
|
902
|
+
None,
|
903
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
|
904
|
+
examples=["my_custom_client_id_key_name"],
|
905
|
+
title="Client ID Key Override",
|
906
|
+
)
|
907
|
+
client_secret_key: Optional[str] = Field(
|
908
|
+
None,
|
909
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
|
910
|
+
examples=["my_custom_client_secret_key_name"],
|
911
|
+
title="Client Secret Key Override",
|
912
|
+
)
|
913
|
+
scope_key: Optional[str] = Field(
|
914
|
+
None,
|
915
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
|
916
|
+
examples=["my_custom_scope_key_key_name"],
|
917
|
+
title="Scopes Key Override",
|
918
|
+
)
|
919
|
+
state_key: Optional[str] = Field(
|
920
|
+
None,
|
921
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.",
|
922
|
+
examples=["my_custom_state_key_key_name"],
|
923
|
+
title="State Key Override",
|
924
|
+
)
|
925
|
+
auth_code_key: Optional[str] = Field(
|
926
|
+
None,
|
927
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.",
|
928
|
+
examples=["my_custom_auth_code_key_name"],
|
929
|
+
title="Auth Code Key Override",
|
930
|
+
)
|
931
|
+
redirect_uri_key: Optional[str] = Field(
|
932
|
+
None,
|
933
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
|
934
|
+
examples=["my_custom_redirect_uri_key_name"],
|
935
|
+
title="Redirect URI Key Override",
|
936
|
+
)
|
481
937
|
|
482
938
|
|
483
939
|
class OAuthConfigSpecification(BaseModel):
|
484
940
|
class Config:
|
485
941
|
extra = Extra.allow
|
486
942
|
|
487
|
-
oauth_user_input_from_connector_config_specification: Optional[
|
488
|
-
Dict[str, Any]
|
489
|
-
] = Field(
|
943
|
+
oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field(
|
490
944
|
None,
|
491
945
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }",
|
492
946
|
examples=[
|
493
|
-
{
|
947
|
+
{"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}},
|
494
948
|
{
|
495
|
-
|
496
|
-
|
497
|
-
|
949
|
+
"app_id": {
|
950
|
+
"type": "string",
|
951
|
+
"path_in_connector_config": ["info", "app_id"],
|
498
952
|
}
|
499
953
|
},
|
500
954
|
],
|
501
|
-
title=
|
955
|
+
title="OAuth user input",
|
956
|
+
)
|
957
|
+
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
|
958
|
+
None,
|
959
|
+
description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{auth_code_key}": "{{auth_code_key}}",\n "{client_id_key}": "{{client_id_key}}",\n "{client_secret_key}": "{{client_secret_key}}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }',
|
960
|
+
title="DeclarativeOAuth Connector Specification",
|
502
961
|
)
|
503
962
|
complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
|
504
963
|
None,
|
505
964
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
|
506
965
|
examples=[
|
507
966
|
{
|
508
|
-
|
509
|
-
|
510
|
-
|
967
|
+
"refresh_token": {
|
968
|
+
"type": "string,",
|
969
|
+
"path_in_connector_config": ["credentials", "refresh_token"],
|
511
970
|
}
|
512
971
|
}
|
513
972
|
],
|
514
|
-
title=
|
973
|
+
title="OAuth output specification",
|
515
974
|
)
|
516
975
|
complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field(
|
517
976
|
None,
|
518
|
-
description=
|
519
|
-
examples=[
|
520
|
-
|
521
|
-
],
|
522
|
-
title='OAuth input specification',
|
977
|
+
description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }",
|
978
|
+
examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}],
|
979
|
+
title="OAuth input specification",
|
523
980
|
)
|
524
981
|
complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field(
|
525
982
|
None,
|
526
983
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }",
|
527
984
|
examples=[
|
528
985
|
{
|
529
|
-
|
530
|
-
|
531
|
-
|
986
|
+
"client_id": {
|
987
|
+
"type": "string,",
|
988
|
+
"path_in_connector_config": ["credentials", "client_id"],
|
532
989
|
},
|
533
|
-
|
534
|
-
|
535
|
-
|
990
|
+
"client_secret": {
|
991
|
+
"type": "string,",
|
992
|
+
"path_in_connector_config": ["credentials", "client_secret"],
|
536
993
|
},
|
537
994
|
}
|
538
995
|
],
|
539
|
-
title=
|
996
|
+
title="OAuth server output specification",
|
540
997
|
)
|
541
998
|
|
542
999
|
|
543
1000
|
class OffsetIncrement(BaseModel):
|
544
|
-
type: Literal[
|
1001
|
+
type: Literal["OffsetIncrement"]
|
545
1002
|
page_size: Optional[Union[int, str]] = Field(
|
546
1003
|
None,
|
547
|
-
description=
|
1004
|
+
description="The number of records to include in each pages.",
|
548
1005
|
examples=[100, "{{ config['page_size'] }}"],
|
549
|
-
title=
|
1006
|
+
title="Limit",
|
550
1007
|
)
|
551
1008
|
inject_on_first_request: Optional[bool] = Field(
|
552
1009
|
False,
|
553
|
-
description=
|
554
|
-
title=
|
1010
|
+
description="Using the `offset` with value `0` during the first request",
|
1011
|
+
title="Inject Offset",
|
555
1012
|
)
|
556
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1013
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
557
1014
|
|
558
1015
|
|
559
1016
|
class PageIncrement(BaseModel):
|
560
|
-
type: Literal[
|
1017
|
+
type: Literal["PageIncrement"]
|
561
1018
|
page_size: Optional[Union[int, str]] = Field(
|
562
1019
|
None,
|
563
|
-
description=
|
564
|
-
examples=[100,
|
565
|
-
title=
|
1020
|
+
description="The number of records to include in each pages.",
|
1021
|
+
examples=[100, "100", "{{ config['page_size'] }}"],
|
1022
|
+
title="Page Size",
|
566
1023
|
)
|
567
1024
|
start_from_page: Optional[int] = Field(
|
568
1025
|
0,
|
569
|
-
description=
|
1026
|
+
description="Index of the first page to request.",
|
570
1027
|
examples=[0, 1],
|
571
|
-
title=
|
1028
|
+
title="Start From Page",
|
572
1029
|
)
|
573
1030
|
inject_on_first_request: Optional[bool] = Field(
|
574
1031
|
False,
|
575
|
-
description=
|
576
|
-
title=
|
1032
|
+
description="Using the `page number` with value defined by `start_from_page` during the first request",
|
1033
|
+
title="Inject Page Number",
|
577
1034
|
)
|
578
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1035
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
579
1036
|
|
580
1037
|
|
581
1038
|
class PrimaryKey(BaseModel):
|
582
1039
|
__root__: Union[str, List[str], List[List[str]]] = Field(
|
583
1040
|
...,
|
584
|
-
description=
|
585
|
-
examples=[
|
586
|
-
title=
|
1041
|
+
description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.",
|
1042
|
+
examples=["id", ["code", "type"]],
|
1043
|
+
title="Primary Key",
|
587
1044
|
)
|
588
1045
|
|
589
1046
|
|
590
1047
|
class RecordFilter(BaseModel):
|
591
|
-
type: Literal[
|
1048
|
+
type: Literal["RecordFilter"]
|
592
1049
|
condition: Optional[str] = Field(
|
593
|
-
|
594
|
-
description=
|
1050
|
+
"",
|
1051
|
+
description="The predicate to filter a record. Records will be removed if evaluated to False.",
|
595
1052
|
examples=[
|
596
1053
|
"{{ record['created_at'] >= stream_interval['start_time'] }}",
|
597
1054
|
"{{ record.status in ['active', 'expired'] }}",
|
598
1055
|
],
|
599
1056
|
)
|
600
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1057
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
601
1058
|
|
602
1059
|
|
603
1060
|
class SchemaNormalization(Enum):
|
604
|
-
None_ =
|
605
|
-
Default =
|
1061
|
+
None_ = "None"
|
1062
|
+
Default = "Default"
|
606
1063
|
|
607
1064
|
|
608
1065
|
class RemoveFields(BaseModel):
|
609
|
-
type: Literal[
|
1066
|
+
type: Literal["RemoveFields"]
|
610
1067
|
condition: Optional[str] = Field(
|
611
|
-
|
612
|
-
description=
|
1068
|
+
"",
|
1069
|
+
description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,",
|
613
1070
|
examples=[
|
614
1071
|
"{{ property|string == '' }}",
|
615
|
-
|
616
|
-
|
1072
|
+
"{{ property is integer }}",
|
1073
|
+
"{{ property|length > 5 }}",
|
617
1074
|
"{{ property == 'some_string_to_match' }}",
|
618
1075
|
],
|
619
1076
|
)
|
620
1077
|
field_pointers: List[List[str]] = Field(
|
621
1078
|
...,
|
622
|
-
description=
|
623
|
-
examples=[[
|
624
|
-
title=
|
1079
|
+
description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.",
|
1080
|
+
examples=[["tags"], [["content", "html"], ["content", "plain_text"]]],
|
1081
|
+
title="Field Paths",
|
625
1082
|
)
|
626
1083
|
|
627
1084
|
|
628
1085
|
class RequestPath(BaseModel):
|
629
|
-
type: Literal[
|
1086
|
+
type: Literal["RequestPath"]
|
630
1087
|
|
631
1088
|
|
632
1089
|
class InjectInto(Enum):
|
633
|
-
request_parameter =
|
634
|
-
header =
|
635
|
-
body_data =
|
636
|
-
body_json =
|
1090
|
+
request_parameter = "request_parameter"
|
1091
|
+
header = "header"
|
1092
|
+
body_data = "body_data"
|
1093
|
+
body_json = "body_json"
|
637
1094
|
|
638
1095
|
|
639
1096
|
class RequestOption(BaseModel):
|
640
|
-
type: Literal[
|
1097
|
+
type: Literal["RequestOption"]
|
641
1098
|
field_name: str = Field(
|
642
1099
|
...,
|
643
|
-
description=
|
644
|
-
examples=[
|
645
|
-
title=
|
1100
|
+
description="Configures which key should be used in the location that the descriptor is being injected into",
|
1101
|
+
examples=["segment_id"],
|
1102
|
+
title="Request Option",
|
646
1103
|
)
|
647
1104
|
inject_into: InjectInto = Field(
|
648
1105
|
...,
|
649
|
-
description=
|
650
|
-
examples=[
|
651
|
-
title=
|
1106
|
+
description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.",
|
1107
|
+
examples=["request_parameter", "header", "body_data", "body_json"],
|
1108
|
+
title="Inject Into",
|
652
1109
|
)
|
653
1110
|
|
654
1111
|
|
@@ -660,106 +1117,184 @@ class Schemas(BaseModel):
|
|
660
1117
|
|
661
1118
|
|
662
1119
|
class LegacySessionTokenAuthenticator(BaseModel):
|
663
|
-
type: Literal[
|
1120
|
+
type: Literal["LegacySessionTokenAuthenticator"]
|
664
1121
|
header: str = Field(
|
665
1122
|
...,
|
666
|
-
description=
|
667
|
-
examples=[
|
668
|
-
title=
|
1123
|
+
description="The name of the session token header that will be injected in the request",
|
1124
|
+
examples=["X-Session"],
|
1125
|
+
title="Session Request Header",
|
669
1126
|
)
|
670
1127
|
login_url: str = Field(
|
671
1128
|
...,
|
672
|
-
description=
|
673
|
-
examples=[
|
674
|
-
title=
|
1129
|
+
description="Path of the login URL (do not include the base URL)",
|
1130
|
+
examples=["session"],
|
1131
|
+
title="Login Path",
|
675
1132
|
)
|
676
1133
|
session_token: Optional[str] = Field(
|
677
1134
|
None,
|
678
|
-
description=
|
1135
|
+
description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair",
|
679
1136
|
example=["{{ config['session_token'] }}"],
|
680
|
-
title=
|
1137
|
+
title="Session Token",
|
681
1138
|
)
|
682
1139
|
session_token_response_key: str = Field(
|
683
1140
|
...,
|
684
|
-
description=
|
685
|
-
examples=[
|
686
|
-
title=
|
1141
|
+
description="Name of the key of the session token to be extracted from the response",
|
1142
|
+
examples=["id"],
|
1143
|
+
title="Response Token Response Key",
|
687
1144
|
)
|
688
1145
|
username: Optional[str] = Field(
|
689
1146
|
None,
|
690
|
-
description=
|
1147
|
+
description="Username used to authenticate and obtain a session token",
|
691
1148
|
examples=[" {{ config['username'] }}"],
|
692
|
-
title=
|
1149
|
+
title="Username",
|
693
1150
|
)
|
694
1151
|
password: Optional[str] = Field(
|
695
|
-
|
696
|
-
description=
|
697
|
-
examples=["{{ config['password'] }}",
|
698
|
-
title=
|
1152
|
+
"",
|
1153
|
+
description="Password used to authenticate and obtain a session token",
|
1154
|
+
examples=["{{ config['password'] }}", ""],
|
1155
|
+
title="Password",
|
699
1156
|
)
|
700
1157
|
validate_session_url: str = Field(
|
701
1158
|
...,
|
702
|
-
description=
|
703
|
-
examples=[
|
704
|
-
title=
|
1159
|
+
description="Path of the URL to use to validate that the session token is valid (do not include the base URL)",
|
1160
|
+
examples=["user/current"],
|
1161
|
+
title="Validate Session Path",
|
705
1162
|
)
|
706
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1163
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1164
|
+
|
1165
|
+
|
1166
|
+
class JsonLineParser(BaseModel):
|
1167
|
+
type: Literal["JsonLineParser"]
|
1168
|
+
encoding: Optional[str] = "utf-8"
|
1169
|
+
|
1170
|
+
|
1171
|
+
class CsvParser(BaseModel):
|
1172
|
+
type: Literal["CsvParser"]
|
1173
|
+
encoding: Optional[str] = "utf-8"
|
1174
|
+
delimiter: Optional[str] = ","
|
1175
|
+
|
1176
|
+
|
1177
|
+
class AsyncJobStatusMap(BaseModel):
|
1178
|
+
type: Optional[Literal["AsyncJobStatusMap"]] = None
|
1179
|
+
running: List[str]
|
1180
|
+
completed: List[str]
|
1181
|
+
failed: List[str]
|
1182
|
+
timeout: List[str]
|
707
1183
|
|
708
1184
|
|
709
1185
|
class ValueType(Enum):
|
710
|
-
string =
|
711
|
-
number =
|
712
|
-
integer =
|
713
|
-
boolean =
|
1186
|
+
string = "string"
|
1187
|
+
number = "number"
|
1188
|
+
integer = "integer"
|
1189
|
+
boolean = "boolean"
|
714
1190
|
|
715
1191
|
|
716
1192
|
class WaitTimeFromHeader(BaseModel):
|
717
|
-
type: Literal[
|
1193
|
+
type: Literal["WaitTimeFromHeader"]
|
718
1194
|
header: str = Field(
|
719
1195
|
...,
|
720
|
-
description=
|
721
|
-
examples=[
|
722
|
-
title=
|
1196
|
+
description="The name of the response header defining how long to wait before retrying.",
|
1197
|
+
examples=["Retry-After"],
|
1198
|
+
title="Response Header Name",
|
723
1199
|
)
|
724
1200
|
regex: Optional[str] = Field(
|
725
1201
|
None,
|
726
|
-
description=
|
727
|
-
examples=[
|
728
|
-
title=
|
1202
|
+
description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
|
1203
|
+
examples=["([-+]?\\d+)"],
|
1204
|
+
title="Extraction Regex",
|
1205
|
+
)
|
1206
|
+
max_waiting_time_in_seconds: Optional[float] = Field(
|
1207
|
+
None,
|
1208
|
+
description="Given the value extracted from the header is greater than this value, stop the stream.",
|
1209
|
+
examples=[3600],
|
1210
|
+
title="Max Waiting Time in Seconds",
|
729
1211
|
)
|
730
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1212
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
731
1213
|
|
732
1214
|
|
733
1215
|
class WaitUntilTimeFromHeader(BaseModel):
|
734
|
-
type: Literal[
|
1216
|
+
type: Literal["WaitUntilTimeFromHeader"]
|
735
1217
|
header: str = Field(
|
736
1218
|
...,
|
737
|
-
description=
|
738
|
-
examples=[
|
739
|
-
title=
|
1219
|
+
description="The name of the response header defining how long to wait before retrying.",
|
1220
|
+
examples=["wait_time"],
|
1221
|
+
title="Response Header",
|
740
1222
|
)
|
741
1223
|
min_wait: Optional[Union[float, str]] = Field(
|
742
1224
|
None,
|
743
|
-
description=
|
744
|
-
examples=[10,
|
745
|
-
title=
|
1225
|
+
description="Minimum time to wait before retrying.",
|
1226
|
+
examples=[10, "60"],
|
1227
|
+
title="Minimum Wait Time",
|
746
1228
|
)
|
747
1229
|
regex: Optional[str] = Field(
|
748
1230
|
None,
|
749
|
-
description=
|
750
|
-
examples=[
|
751
|
-
title=
|
1231
|
+
description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
|
1232
|
+
examples=["([-+]?\\d+)"],
|
1233
|
+
title="Extraction Regex",
|
752
1234
|
)
|
753
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1235
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1236
|
+
|
1237
|
+
|
1238
|
+
class ComponentMappingDefinition(BaseModel):
|
1239
|
+
type: Literal["ComponentMappingDefinition"]
|
1240
|
+
field_path: List[str] = Field(
|
1241
|
+
...,
|
1242
|
+
description="A list of potentially nested fields indicating the full path where value will be added or updated.",
|
1243
|
+
examples=[
|
1244
|
+
["data"],
|
1245
|
+
["data", "records"],
|
1246
|
+
["data", 1, "name"],
|
1247
|
+
["data", "{{ components_values.name }}"],
|
1248
|
+
["data", "*", "record"],
|
1249
|
+
["*", "**", "name"],
|
1250
|
+
],
|
1251
|
+
title="Field Path",
|
1252
|
+
)
|
1253
|
+
value: str = Field(
|
1254
|
+
...,
|
1255
|
+
description="The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.",
|
1256
|
+
examples=[
|
1257
|
+
"{{ components_values['updates'] }}",
|
1258
|
+
"{{ components_values['MetaData']['LastUpdatedTime'] }}",
|
1259
|
+
"{{ config['segment_id'] }}",
|
1260
|
+
"{{ stream_slice['parent_id'] }}",
|
1261
|
+
"{{ stream_slice['extra_fields']['name'] }}",
|
1262
|
+
],
|
1263
|
+
title="Value",
|
1264
|
+
)
|
1265
|
+
value_type: Optional[ValueType] = Field(
|
1266
|
+
None,
|
1267
|
+
description="The expected data type of the value. If omitted, the type will be inferred from the value provided.",
|
1268
|
+
title="Value Type",
|
1269
|
+
)
|
1270
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1271
|
+
|
1272
|
+
|
1273
|
+
class StreamConfig(BaseModel):
|
1274
|
+
type: Literal["StreamConfig"]
|
1275
|
+
configs_pointer: List[str] = Field(
|
1276
|
+
...,
|
1277
|
+
description="A list of potentially nested fields indicating the full path in source config file where streams configs located.",
|
1278
|
+
examples=[["data"], ["data", "streams"], ["data", "{{ parameters.name }}"]],
|
1279
|
+
title="Configs Pointer",
|
1280
|
+
)
|
1281
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1282
|
+
|
1283
|
+
|
1284
|
+
class ConfigComponentsResolver(BaseModel):
|
1285
|
+
type: Literal["ConfigComponentsResolver"]
|
1286
|
+
stream_config: StreamConfig
|
1287
|
+
components_mapping: List[ComponentMappingDefinition]
|
1288
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
754
1289
|
|
755
1290
|
|
756
1291
|
class AddedFieldDefinition(BaseModel):
|
757
|
-
type: Literal[
|
1292
|
+
type: Literal["AddedFieldDefinition"]
|
758
1293
|
path: List[str] = Field(
|
759
1294
|
...,
|
760
|
-
description=
|
761
|
-
examples=[[
|
762
|
-
title=
|
1295
|
+
description="List of strings defining the path where to add the value on the record.",
|
1296
|
+
examples=[["segment_id"], ["metadata", "segment_id"]],
|
1297
|
+
title="Path",
|
763
1298
|
)
|
764
1299
|
value: str = Field(
|
765
1300
|
...,
|
@@ -769,187 +1304,167 @@ class AddedFieldDefinition(BaseModel):
|
|
769
1304
|
"{{ record['MetaData']['LastUpdatedTime'] }}",
|
770
1305
|
"{{ stream_partition['segment_id'] }}",
|
771
1306
|
],
|
772
|
-
title=
|
1307
|
+
title="Value",
|
773
1308
|
)
|
774
1309
|
value_type: Optional[ValueType] = Field(
|
775
1310
|
None,
|
776
|
-
description=
|
777
|
-
title=
|
1311
|
+
description="Type of the value. If not specified, the type will be inferred from the value.",
|
1312
|
+
title="Value Type",
|
778
1313
|
)
|
779
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1314
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
780
1315
|
|
781
1316
|
|
782
1317
|
class AddFields(BaseModel):
|
783
|
-
type: Literal[
|
1318
|
+
type: Literal["AddFields"]
|
784
1319
|
fields: List[AddedFieldDefinition] = Field(
|
785
1320
|
...,
|
786
|
-
description=
|
787
|
-
title=
|
1321
|
+
description="List of transformations (path and corresponding value) that will be added to the record.",
|
1322
|
+
title="Fields",
|
788
1323
|
)
|
789
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1324
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
790
1325
|
|
791
1326
|
|
792
1327
|
class ApiKeyAuthenticator(BaseModel):
|
793
|
-
type: Literal[
|
1328
|
+
type: Literal["ApiKeyAuthenticator"]
|
794
1329
|
api_token: Optional[str] = Field(
|
795
1330
|
None,
|
796
|
-
description=
|
1331
|
+
description="The API key to inject in the request. Fill it in the user inputs.",
|
797
1332
|
examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"],
|
798
|
-
title=
|
1333
|
+
title="API Key",
|
799
1334
|
)
|
800
1335
|
header: Optional[str] = Field(
|
801
1336
|
None,
|
802
|
-
description=
|
803
|
-
examples=[
|
804
|
-
title=
|
1337
|
+
description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.",
|
1338
|
+
examples=["Authorization", "Api-Token", "X-Auth-Token"],
|
1339
|
+
title="Header Name",
|
805
1340
|
)
|
806
1341
|
inject_into: Optional[RequestOption] = Field(
|
807
1342
|
None,
|
808
|
-
description=
|
1343
|
+
description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.",
|
809
1344
|
examples=[
|
810
|
-
{
|
811
|
-
{
|
1345
|
+
{"inject_into": "header", "field_name": "Authorization"},
|
1346
|
+
{"inject_into": "request_parameter", "field_name": "authKey"},
|
812
1347
|
],
|
813
|
-
title=
|
1348
|
+
title="Inject API Key Into Outgoing HTTP Request",
|
814
1349
|
)
|
815
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1350
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
816
1351
|
|
817
1352
|
|
818
1353
|
class AuthFlow(BaseModel):
|
819
1354
|
auth_flow_type: Optional[AuthFlowType] = Field(
|
820
|
-
None, description=
|
1355
|
+
None, description="The type of auth to use", title="Auth flow type"
|
821
1356
|
)
|
822
1357
|
predicate_key: Optional[List[str]] = Field(
|
823
1358
|
None,
|
824
|
-
description=
|
825
|
-
examples=[[
|
826
|
-
title=
|
1359
|
+
description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.",
|
1360
|
+
examples=[["credentials", "auth_type"]],
|
1361
|
+
title="Predicate key",
|
827
1362
|
)
|
828
1363
|
predicate_value: Optional[str] = Field(
|
829
1364
|
None,
|
830
|
-
description=
|
831
|
-
examples=[
|
832
|
-
title=
|
1365
|
+
description="Value of the predicate_key fields for the advanced auth to be applicable.",
|
1366
|
+
examples=["Oauth"],
|
1367
|
+
title="Predicate value",
|
833
1368
|
)
|
834
1369
|
oauth_config_specification: Optional[OAuthConfigSpecification] = None
|
835
1370
|
|
836
1371
|
|
837
|
-
class CursorPagination(BaseModel):
|
838
|
-
type: Literal['CursorPagination']
|
839
|
-
cursor_value: str = Field(
|
840
|
-
...,
|
841
|
-
description='Value of the cursor defining the next page to fetch.',
|
842
|
-
examples=[
|
843
|
-
'{{ headers.link.next.cursor }}',
|
844
|
-
"{{ last_records[-1]['key'] }}",
|
845
|
-
"{{ response['nextPage'] }}",
|
846
|
-
],
|
847
|
-
title='Cursor Value',
|
848
|
-
)
|
849
|
-
page_size: Optional[int] = Field(
|
850
|
-
None,
|
851
|
-
description='The number of records to include in each pages.',
|
852
|
-
examples=[100],
|
853
|
-
title='Page Size',
|
854
|
-
)
|
855
|
-
stop_condition: Optional[str] = Field(
|
856
|
-
None,
|
857
|
-
description='Template string evaluating when to stop paginating.',
|
858
|
-
examples=[
|
859
|
-
'{{ response.data.has_more is false }}',
|
860
|
-
"{{ 'next' not in headers['link'] }}",
|
861
|
-
],
|
862
|
-
title='Stop Condition',
|
863
|
-
)
|
864
|
-
decoder: Optional[JsonDecoder] = Field(
|
865
|
-
None,
|
866
|
-
description='Component decoding the response so records can be extracted.',
|
867
|
-
title='Decoder',
|
868
|
-
)
|
869
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
870
|
-
|
871
|
-
|
872
1372
|
class DatetimeBasedCursor(BaseModel):
|
873
|
-
type: Literal[
|
1373
|
+
type: Literal["DatetimeBasedCursor"]
|
874
1374
|
cursor_field: str = Field(
|
875
1375
|
...,
|
876
|
-
description=
|
877
|
-
examples=[
|
878
|
-
title=
|
1376
|
+
description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.",
|
1377
|
+
examples=["created_at", "{{ config['record_cursor'] }}"],
|
1378
|
+
title="Cursor Field",
|
879
1379
|
)
|
880
1380
|
datetime_format: str = Field(
|
881
1381
|
...,
|
882
|
-
description=
|
883
|
-
examples=[
|
884
|
-
title=
|
1382
|
+
description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
|
1383
|
+
examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"],
|
1384
|
+
title="Outgoing Datetime Format",
|
885
1385
|
)
|
886
1386
|
start_datetime: Union[str, MinMaxDatetime] = Field(
|
887
1387
|
...,
|
888
|
-
description=
|
889
|
-
examples=[
|
890
|
-
title=
|
1388
|
+
description="The datetime that determines the earliest record that should be synced.",
|
1389
|
+
examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"],
|
1390
|
+
title="Start Datetime",
|
891
1391
|
)
|
892
1392
|
cursor_datetime_formats: Optional[List[str]] = Field(
|
893
1393
|
None,
|
894
|
-
description=
|
895
|
-
title=
|
1394
|
+
description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.",
|
1395
|
+
title="Cursor Datetime Formats",
|
896
1396
|
)
|
897
1397
|
cursor_granularity: Optional[str] = Field(
|
898
1398
|
None,
|
899
|
-
description=
|
900
|
-
examples=[
|
901
|
-
title=
|
1399
|
+
description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.",
|
1400
|
+
examples=["PT1S"],
|
1401
|
+
title="Cursor Granularity",
|
902
1402
|
)
|
903
1403
|
end_datetime: Optional[Union[str, MinMaxDatetime]] = Field(
|
904
1404
|
None,
|
905
|
-
description=
|
906
|
-
examples=[
|
907
|
-
title=
|
1405
|
+
description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.",
|
1406
|
+
examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"],
|
1407
|
+
title="End Datetime",
|
908
1408
|
)
|
909
1409
|
end_time_option: Optional[RequestOption] = Field(
|
910
1410
|
None,
|
911
|
-
description=
|
912
|
-
title=
|
1411
|
+
description="Optionally configures how the end datetime will be sent in requests to the source API.",
|
1412
|
+
title="Inject End Time Into Outgoing HTTP Request",
|
913
1413
|
)
|
914
1414
|
is_data_feed: Optional[bool] = Field(
|
915
1415
|
None,
|
916
|
-
description=
|
917
|
-
title=
|
1416
|
+
description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.",
|
1417
|
+
title="Whether the target API is formatted as a data feed",
|
1418
|
+
)
|
1419
|
+
is_client_side_incremental: Optional[bool] = Field(
|
1420
|
+
None,
|
1421
|
+
description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.",
|
1422
|
+
title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)",
|
1423
|
+
)
|
1424
|
+
is_compare_strictly: Optional[bool] = Field(
|
1425
|
+
False,
|
1426
|
+
description="Set to True if the target API does not accept queries where the start time equal the end time.",
|
1427
|
+
title="Whether to skip requests if the start time equals the end time",
|
1428
|
+
)
|
1429
|
+
global_substream_cursor: Optional[bool] = Field(
|
1430
|
+
False,
|
1431
|
+
description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).",
|
1432
|
+
title="Whether to store cursor as one value instead of per partition",
|
918
1433
|
)
|
919
1434
|
lookback_window: Optional[str] = Field(
|
920
1435
|
None,
|
921
|
-
description=
|
922
|
-
examples=[
|
923
|
-
title=
|
1436
|
+
description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.",
|
1437
|
+
examples=["P1D", "P{{ config['lookback_days'] }}D"],
|
1438
|
+
title="Lookback Window",
|
924
1439
|
)
|
925
1440
|
partition_field_end: Optional[str] = Field(
|
926
1441
|
None,
|
927
|
-
description=
|
928
|
-
examples=[
|
929
|
-
title=
|
1442
|
+
description="Name of the partition start time field.",
|
1443
|
+
examples=["ending_time"],
|
1444
|
+
title="Partition Field End",
|
930
1445
|
)
|
931
1446
|
partition_field_start: Optional[str] = Field(
|
932
1447
|
None,
|
933
|
-
description=
|
934
|
-
examples=[
|
935
|
-
title=
|
1448
|
+
description="Name of the partition end time field.",
|
1449
|
+
examples=["starting_time"],
|
1450
|
+
title="Partition Field Start",
|
936
1451
|
)
|
937
1452
|
start_time_option: Optional[RequestOption] = Field(
|
938
1453
|
None,
|
939
|
-
description=
|
940
|
-
title=
|
1454
|
+
description="Optionally configures how the start datetime will be sent in requests to the source API.",
|
1455
|
+
title="Inject Start Time Into Outgoing HTTP Request",
|
941
1456
|
)
|
942
1457
|
step: Optional[str] = Field(
|
943
1458
|
None,
|
944
|
-
description=
|
945
|
-
examples=[
|
946
|
-
title=
|
1459
|
+
description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.",
|
1460
|
+
examples=["P1W", "{{ config['step_increment'] }}"],
|
1461
|
+
title="Step",
|
947
1462
|
)
|
948
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1463
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
949
1464
|
|
950
1465
|
|
951
1466
|
class DefaultErrorHandler(BaseModel):
|
952
|
-
type: Literal[
|
1467
|
+
type: Literal["DefaultErrorHandler"]
|
953
1468
|
backoff_strategies: Optional[
|
954
1469
|
List[
|
955
1470
|
Union[
|
@@ -962,154 +1477,188 @@ class DefaultErrorHandler(BaseModel):
|
|
962
1477
|
]
|
963
1478
|
] = Field(
|
964
1479
|
None,
|
965
|
-
description=
|
966
|
-
title=
|
1480
|
+
description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.",
|
1481
|
+
title="Backoff Strategies",
|
967
1482
|
)
|
968
1483
|
max_retries: Optional[int] = Field(
|
969
1484
|
5,
|
970
|
-
description=
|
1485
|
+
description="The maximum number of time to retry a retryable request before giving up and failing.",
|
971
1486
|
examples=[5, 0, 10],
|
972
|
-
title=
|
1487
|
+
title="Max Retry Count",
|
973
1488
|
)
|
974
1489
|
response_filters: Optional[List[HttpResponseFilter]] = Field(
|
975
1490
|
None,
|
976
1491
|
description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.",
|
977
|
-
title=
|
1492
|
+
title="Response Filters",
|
978
1493
|
)
|
979
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1494
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
980
1495
|
|
981
1496
|
|
982
1497
|
class DefaultPaginator(BaseModel):
|
983
|
-
type: Literal[
|
1498
|
+
type: Literal["DefaultPaginator"]
|
984
1499
|
pagination_strategy: Union[
|
985
1500
|
CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement
|
986
1501
|
] = Field(
|
987
1502
|
...,
|
988
|
-
description=
|
989
|
-
title=
|
990
|
-
)
|
991
|
-
decoder: Optional[JsonDecoder] = Field(
|
992
|
-
None,
|
993
|
-
description='Component decoding the response so records can be extracted.',
|
994
|
-
title='Decoder',
|
1503
|
+
description="Strategy defining how records are paginated.",
|
1504
|
+
title="Pagination Strategy",
|
995
1505
|
)
|
996
1506
|
page_size_option: Optional[RequestOption] = None
|
997
1507
|
page_token_option: Optional[Union[RequestOption, RequestPath]] = None
|
998
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
999
|
-
|
1000
|
-
|
1001
|
-
class DpathExtractor(BaseModel):
|
1002
|
-
type: Literal['DpathExtractor']
|
1003
|
-
field_path: List[str] = Field(
|
1004
|
-
...,
|
1005
|
-
description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
|
1006
|
-
examples=[
|
1007
|
-
['data'],
|
1008
|
-
['data', 'records'],
|
1009
|
-
['data', '{{ parameters.name }}'],
|
1010
|
-
['data', '*', 'record'],
|
1011
|
-
],
|
1012
|
-
title='Field Path',
|
1013
|
-
)
|
1014
|
-
decoder: Optional[JsonDecoder] = Field(
|
1015
|
-
None,
|
1016
|
-
description='Component decoding the response so records can be extracted.',
|
1017
|
-
title='Decoder',
|
1018
|
-
)
|
1019
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1508
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1020
1509
|
|
1021
1510
|
|
1022
1511
|
class SessionTokenRequestApiKeyAuthenticator(BaseModel):
|
1023
|
-
type: Literal[
|
1512
|
+
type: Literal["ApiKey"]
|
1024
1513
|
inject_into: RequestOption = Field(
|
1025
1514
|
...,
|
1026
|
-
description=
|
1515
|
+
description="Configure how the API Key will be sent in requests to the source API.",
|
1027
1516
|
examples=[
|
1028
|
-
{
|
1029
|
-
{
|
1517
|
+
{"inject_into": "header", "field_name": "Authorization"},
|
1518
|
+
{"inject_into": "request_parameter", "field_name": "authKey"},
|
1030
1519
|
],
|
1031
|
-
title=
|
1520
|
+
title="Inject API Key Into Outgoing HTTP Request",
|
1032
1521
|
)
|
1033
1522
|
|
1034
1523
|
|
1035
1524
|
class ListPartitionRouter(BaseModel):
|
1036
|
-
type: Literal[
|
1525
|
+
type: Literal["ListPartitionRouter"]
|
1037
1526
|
cursor_field: str = Field(
|
1038
1527
|
...,
|
1039
1528
|
description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.',
|
1040
|
-
examples=[
|
1041
|
-
title=
|
1529
|
+
examples=["section", "{{ config['section_key'] }}"],
|
1530
|
+
title="Current Partition Value Identifier",
|
1042
1531
|
)
|
1043
1532
|
values: Union[str, List[str]] = Field(
|
1044
1533
|
...,
|
1045
|
-
description=
|
1046
|
-
examples=[[
|
1047
|
-
title=
|
1534
|
+
description="The list of attributes being iterated over and used as input for the requests made to the source API.",
|
1535
|
+
examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"],
|
1536
|
+
title="Partition Values",
|
1048
1537
|
)
|
1049
1538
|
request_option: Optional[RequestOption] = Field(
|
1050
1539
|
None,
|
1051
|
-
description=
|
1052
|
-
title=
|
1540
|
+
description="A request option describing where the list value should be injected into and under what field name if applicable.",
|
1541
|
+
title="Inject Partition Value Into Outgoing HTTP Request",
|
1053
1542
|
)
|
1054
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1543
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1055
1544
|
|
1056
1545
|
|
1057
1546
|
class RecordSelector(BaseModel):
|
1058
|
-
type: Literal[
|
1547
|
+
type: Literal["RecordSelector"]
|
1059
1548
|
extractor: Union[CustomRecordExtractor, DpathExtractor]
|
1060
1549
|
record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field(
|
1061
1550
|
None,
|
1062
|
-
description=
|
1063
|
-
title=
|
1551
|
+
description="Responsible for filtering records to be emitted by the Source.",
|
1552
|
+
title="Record Filter",
|
1553
|
+
)
|
1554
|
+
schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field(
|
1555
|
+
SchemaNormalization.None_,
|
1556
|
+
description="Responsible for normalization according to the schema.",
|
1557
|
+
title="Schema Normalization",
|
1064
1558
|
)
|
1065
|
-
|
1066
|
-
|
1559
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1560
|
+
|
1561
|
+
|
1562
|
+
class GzipParser(BaseModel):
|
1563
|
+
type: Literal["GzipParser"]
|
1564
|
+
inner_parser: Union[JsonLineParser, CsvParser]
|
1067
1565
|
|
1068
1566
|
|
1069
1567
|
class Spec(BaseModel):
|
1070
|
-
type: Literal[
|
1568
|
+
type: Literal["Spec"]
|
1071
1569
|
connection_specification: Dict[str, Any] = Field(
|
1072
1570
|
...,
|
1073
|
-
description=
|
1074
|
-
title=
|
1571
|
+
description="A connection specification describing how a the connector can be configured.",
|
1572
|
+
title="Connection Specification",
|
1075
1573
|
)
|
1076
1574
|
documentation_url: Optional[str] = Field(
|
1077
1575
|
None,
|
1078
1576
|
description="URL of the connector's documentation page.",
|
1079
|
-
examples=[
|
1080
|
-
title=
|
1577
|
+
examples=["https://docs.airbyte.com/integrations/sources/dremio"],
|
1578
|
+
title="Documentation URL",
|
1081
1579
|
)
|
1082
1580
|
advanced_auth: Optional[AuthFlow] = Field(
|
1083
1581
|
None,
|
1084
|
-
description=
|
1085
|
-
title=
|
1582
|
+
description="Advanced specification for configuring the authentication flow.",
|
1583
|
+
title="Advanced Auth",
|
1086
1584
|
)
|
1087
1585
|
|
1088
1586
|
|
1089
1587
|
class CompositeErrorHandler(BaseModel):
|
1090
|
-
type: Literal[
|
1588
|
+
type: Literal["CompositeErrorHandler"]
|
1091
1589
|
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field(
|
1092
1590
|
...,
|
1093
|
-
description=
|
1094
|
-
title=
|
1591
|
+
description="List of error handlers to iterate on to determine how to handle a failed response.",
|
1592
|
+
title="Error Handlers",
|
1095
1593
|
)
|
1096
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1594
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1097
1595
|
|
1098
1596
|
|
1099
|
-
class
|
1597
|
+
class CompositeRawDecoder(BaseModel):
|
1598
|
+
type: Literal["CompositeRawDecoder"]
|
1599
|
+
parser: Union[GzipParser, JsonLineParser, CsvParser]
|
1600
|
+
|
1601
|
+
|
1602
|
+
class DeclarativeSource1(BaseModel):
|
1100
1603
|
class Config:
|
1101
1604
|
extra = Extra.forbid
|
1102
1605
|
|
1103
|
-
type: Literal[
|
1606
|
+
type: Literal["DeclarativeSource"]
|
1104
1607
|
check: CheckStream
|
1105
1608
|
streams: List[DeclarativeStream]
|
1106
|
-
|
1609
|
+
dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None
|
1610
|
+
version: str = Field(
|
1611
|
+
...,
|
1612
|
+
description="The version of the Airbyte CDK used to build and test the source.",
|
1613
|
+
)
|
1614
|
+
schemas: Optional[Schemas] = None
|
1615
|
+
definitions: Optional[Dict[str, Any]] = None
|
1616
|
+
spec: Optional[Spec] = None
|
1617
|
+
concurrency_level: Optional[ConcurrencyLevel] = None
|
1618
|
+
metadata: Optional[Dict[str, Any]] = Field(
|
1619
|
+
None,
|
1620
|
+
description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
|
1621
|
+
)
|
1622
|
+
description: Optional[str] = Field(
|
1623
|
+
None,
|
1624
|
+
description="A description of the connector. It will be presented on the Source documentation page.",
|
1625
|
+
)
|
1626
|
+
|
1627
|
+
|
1628
|
+
class DeclarativeSource2(BaseModel):
|
1629
|
+
class Config:
|
1630
|
+
extra = Extra.forbid
|
1631
|
+
|
1632
|
+
type: Literal["DeclarativeSource"]
|
1633
|
+
check: CheckStream
|
1634
|
+
streams: Optional[List[DeclarativeStream]] = None
|
1635
|
+
dynamic_streams: List[DynamicDeclarativeStream]
|
1636
|
+
version: str = Field(
|
1637
|
+
...,
|
1638
|
+
description="The version of the Airbyte CDK used to build and test the source.",
|
1639
|
+
)
|
1107
1640
|
schemas: Optional[Schemas] = None
|
1108
1641
|
definitions: Optional[Dict[str, Any]] = None
|
1109
1642
|
spec: Optional[Spec] = None
|
1643
|
+
concurrency_level: Optional[ConcurrencyLevel] = None
|
1110
1644
|
metadata: Optional[Dict[str, Any]] = Field(
|
1111
1645
|
None,
|
1112
|
-
description=
|
1646
|
+
description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
|
1647
|
+
)
|
1648
|
+
description: Optional[str] = Field(
|
1649
|
+
None,
|
1650
|
+
description="A description of the connector. It will be presented on the Source documentation page.",
|
1651
|
+
)
|
1652
|
+
|
1653
|
+
|
1654
|
+
class DeclarativeSource(BaseModel):
|
1655
|
+
class Config:
|
1656
|
+
extra = Extra.forbid
|
1657
|
+
|
1658
|
+
__root__: Union[DeclarativeSource1, DeclarativeSource2] = Field(
|
1659
|
+
...,
|
1660
|
+
description="An API source that extracts data according to its declarative components.",
|
1661
|
+
title="DeclarativeSource",
|
1113
1662
|
)
|
1114
1663
|
|
1115
1664
|
|
@@ -1117,12 +1666,12 @@ class SelectiveAuthenticator(BaseModel):
|
|
1117
1666
|
class Config:
|
1118
1667
|
extra = Extra.allow
|
1119
1668
|
|
1120
|
-
type: Literal[
|
1669
|
+
type: Literal["SelectiveAuthenticator"]
|
1121
1670
|
authenticator_selection_path: List[str] = Field(
|
1122
1671
|
...,
|
1123
|
-
description=
|
1124
|
-
examples=[[
|
1125
|
-
title=
|
1672
|
+
description="Path of the field in config with selected authenticator name",
|
1673
|
+
examples=[["auth"], ["auth", "type"]],
|
1674
|
+
title="Authenticator Selection Path",
|
1126
1675
|
)
|
1127
1676
|
authenticators: Dict[
|
1128
1677
|
str,
|
@@ -1132,127 +1681,150 @@ class SelectiveAuthenticator(BaseModel):
|
|
1132
1681
|
BearerAuthenticator,
|
1133
1682
|
CustomAuthenticator,
|
1134
1683
|
OAuthAuthenticator,
|
1684
|
+
JwtAuthenticator,
|
1135
1685
|
NoAuth,
|
1136
1686
|
SessionTokenAuthenticator,
|
1137
1687
|
LegacySessionTokenAuthenticator,
|
1138
1688
|
],
|
1139
1689
|
] = Field(
|
1140
1690
|
...,
|
1141
|
-
description=
|
1691
|
+
description="Authenticators to select from.",
|
1142
1692
|
examples=[
|
1143
1693
|
{
|
1144
|
-
|
1145
|
-
|
1146
|
-
|
1694
|
+
"authenticators": {
|
1695
|
+
"token": "#/definitions/ApiKeyAuthenticator",
|
1696
|
+
"oauth": "#/definitions/OAuthAuthenticator",
|
1697
|
+
"jwt": "#/definitions/JwtAuthenticator",
|
1147
1698
|
}
|
1148
1699
|
}
|
1149
1700
|
],
|
1150
|
-
title=
|
1701
|
+
title="Authenticators",
|
1151
1702
|
)
|
1152
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1703
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1153
1704
|
|
1154
1705
|
|
1155
1706
|
class DeclarativeStream(BaseModel):
|
1156
1707
|
class Config:
|
1157
1708
|
extra = Extra.allow
|
1158
1709
|
|
1159
|
-
type: Literal[
|
1160
|
-
retriever: Union[CustomRetriever, SimpleRetriever] = Field(
|
1710
|
+
type: Literal["DeclarativeStream"]
|
1711
|
+
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
|
1161
1712
|
...,
|
1162
|
-
description=
|
1163
|
-
title=
|
1713
|
+
description="Component used to coordinate how records are extracted across stream slices and request pages.",
|
1714
|
+
title="Retriever",
|
1164
1715
|
)
|
1165
|
-
incremental_sync: Optional[
|
1166
|
-
Union[CustomIncrementalSync, DatetimeBasedCursor]
|
1167
|
-
] = Field(
|
1716
|
+
incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field(
|
1168
1717
|
None,
|
1169
|
-
description=
|
1170
|
-
title=
|
1171
|
-
)
|
1172
|
-
name: Optional[str] = Field(
|
1173
|
-
'', description='The stream name.', example=['Users'], title='Name'
|
1718
|
+
description="Component used to fetch data incrementally based on a time field in the data.",
|
1719
|
+
title="Incremental Sync",
|
1174
1720
|
)
|
1721
|
+
name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name")
|
1175
1722
|
primary_key: Optional[PrimaryKey] = Field(
|
1176
|
-
|
1723
|
+
"", description="The primary key of the stream.", title="Primary Key"
|
1177
1724
|
)
|
1178
1725
|
schema_loader: Optional[
|
1179
|
-
Union[
|
1726
|
+
Union[
|
1727
|
+
DynamicSchemaLoader,
|
1728
|
+
InlineSchemaLoader,
|
1729
|
+
JsonFileSchemaLoader,
|
1730
|
+
CustomSchemaLoader,
|
1731
|
+
]
|
1180
1732
|
] = Field(
|
1181
1733
|
None,
|
1182
|
-
description=
|
1183
|
-
title=
|
1734
|
+
description="Component used to retrieve the schema for the current stream.",
|
1735
|
+
title="Schema Loader",
|
1184
1736
|
)
|
1185
1737
|
transformations: Optional[
|
1186
|
-
List[
|
1738
|
+
List[
|
1739
|
+
Union[
|
1740
|
+
AddFields,
|
1741
|
+
CustomTransformation,
|
1742
|
+
RemoveFields,
|
1743
|
+
KeysToLower,
|
1744
|
+
KeysToSnakeCase,
|
1745
|
+
FlattenFields,
|
1746
|
+
KeysReplace,
|
1747
|
+
]
|
1748
|
+
]
|
1187
1749
|
] = Field(
|
1188
1750
|
None,
|
1189
|
-
description=
|
1190
|
-
title=
|
1751
|
+
description="A list of transformations to be applied to each output record.",
|
1752
|
+
title="Transformations",
|
1191
1753
|
)
|
1192
|
-
|
1754
|
+
state_migrations: Optional[
|
1755
|
+
List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]
|
1756
|
+
] = Field(
|
1757
|
+
[],
|
1758
|
+
description="Array of state migrations to be applied on the input state",
|
1759
|
+
title="State Migrations",
|
1760
|
+
)
|
1761
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1193
1762
|
|
1194
1763
|
|
1195
1764
|
class SessionTokenAuthenticator(BaseModel):
|
1196
|
-
type: Literal[
|
1765
|
+
type: Literal["SessionTokenAuthenticator"]
|
1197
1766
|
login_requester: HttpRequester = Field(
|
1198
1767
|
...,
|
1199
|
-
description=
|
1768
|
+
description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.",
|
1200
1769
|
examples=[
|
1201
1770
|
{
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
1207
|
-
|
1208
|
-
|
1771
|
+
"type": "HttpRequester",
|
1772
|
+
"url_base": "https://my_api.com",
|
1773
|
+
"path": "/login",
|
1774
|
+
"authenticator": {
|
1775
|
+
"type": "BasicHttpAuthenticator",
|
1776
|
+
"username": "{{ config.username }}",
|
1777
|
+
"password": "{{ config.password }}",
|
1209
1778
|
},
|
1210
1779
|
}
|
1211
1780
|
],
|
1212
|
-
title=
|
1781
|
+
title="Login Requester",
|
1213
1782
|
)
|
1214
1783
|
session_token_path: List[str] = Field(
|
1215
1784
|
...,
|
1216
|
-
description=
|
1217
|
-
examples=[[
|
1218
|
-
title=
|
1785
|
+
description="The path in the response body returned from the login requester to the session token.",
|
1786
|
+
examples=[["access_token"], ["result", "token"]],
|
1787
|
+
title="Session Token Path",
|
1219
1788
|
)
|
1220
1789
|
expiration_duration: Optional[str] = Field(
|
1221
1790
|
None,
|
1222
|
-
description=
|
1223
|
-
examples=[
|
1224
|
-
title=
|
1791
|
+
description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.",
|
1792
|
+
examples=["PT1H", "P1D"],
|
1793
|
+
title="Expiration Duration",
|
1225
1794
|
)
|
1226
1795
|
request_authentication: Union[
|
1227
1796
|
SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator
|
1228
1797
|
] = Field(
|
1229
1798
|
...,
|
1230
|
-
description=
|
1231
|
-
title=
|
1799
|
+
description="Authentication method to use for requests sent to the API, specifying how to inject the session token.",
|
1800
|
+
title="Data Request Authentication",
|
1232
1801
|
)
|
1233
|
-
|
1802
|
+
decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field(
|
1803
|
+
None, description="Component used to decode the response.", title="Decoder"
|
1804
|
+
)
|
1805
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1234
1806
|
|
1235
1807
|
|
1236
1808
|
class HttpRequester(BaseModel):
|
1237
|
-
type: Literal[
|
1809
|
+
type: Literal["HttpRequester"]
|
1238
1810
|
url_base: str = Field(
|
1239
1811
|
...,
|
1240
|
-
description=
|
1812
|
+
description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
|
1241
1813
|
examples=[
|
1242
|
-
|
1814
|
+
"https://connect.squareup.com/v2",
|
1243
1815
|
"{{ config['base_url'] or 'https://app.posthog.com'}}/api/",
|
1244
1816
|
],
|
1245
|
-
title=
|
1817
|
+
title="API Base URL",
|
1246
1818
|
)
|
1247
1819
|
path: str = Field(
|
1248
1820
|
...,
|
1249
|
-
description=
|
1821
|
+
description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
|
1250
1822
|
examples=[
|
1251
|
-
|
1823
|
+
"/products",
|
1252
1824
|
"/quotes/{{ stream_partition['id'] }}/quote_line_groups",
|
1253
1825
|
"/trades/{{ config['symbol_id'] }}/history",
|
1254
1826
|
],
|
1255
|
-
title=
|
1827
|
+
title="URL Path",
|
1256
1828
|
)
|
1257
1829
|
authenticator: Optional[
|
1258
1830
|
Union[
|
@@ -1261,6 +1833,7 @@ class HttpRequester(BaseModel):
|
|
1261
1833
|
BearerAuthenticator,
|
1262
1834
|
CustomAuthenticator,
|
1263
1835
|
OAuthAuthenticator,
|
1836
|
+
JwtAuthenticator,
|
1264
1837
|
NoAuth,
|
1265
1838
|
SessionTokenAuthenticator,
|
1266
1839
|
LegacySessionTokenAuthenticator,
|
@@ -1268,101 +1841,139 @@ class HttpRequester(BaseModel):
|
|
1268
1841
|
]
|
1269
1842
|
] = Field(
|
1270
1843
|
None,
|
1271
|
-
description=
|
1272
|
-
title=
|
1844
|
+
description="Authentication method to use for requests sent to the API.",
|
1845
|
+
title="Authenticator",
|
1273
1846
|
)
|
1274
1847
|
error_handler: Optional[
|
1275
1848
|
Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]
|
1276
1849
|
] = Field(
|
1277
1850
|
None,
|
1278
|
-
description=
|
1279
|
-
title=
|
1851
|
+
description="Error handler component that defines how to handle errors.",
|
1852
|
+
title="Error Handler",
|
1280
1853
|
)
|
1281
1854
|
http_method: Optional[HttpMethod] = Field(
|
1282
1855
|
HttpMethod.GET,
|
1283
|
-
description=
|
1284
|
-
examples=[
|
1285
|
-
title=
|
1856
|
+
description="The HTTP method used to fetch data from the source (can be GET or POST).",
|
1857
|
+
examples=["GET", "POST"],
|
1858
|
+
title="HTTP Method",
|
1286
1859
|
)
|
1287
1860
|
request_body_data: Optional[Union[str, Dict[str, str]]] = Field(
|
1288
1861
|
None,
|
1289
|
-
description=
|
1862
|
+
description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.",
|
1290
1863
|
examples=[
|
1291
1864
|
'[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n'
|
1292
1865
|
],
|
1293
|
-
title=
|
1866
|
+
title="Request Body Payload (Non-JSON)",
|
1294
1867
|
)
|
1295
1868
|
request_body_json: Optional[Union[str, Dict[str, Any]]] = Field(
|
1296
1869
|
None,
|
1297
|
-
description=
|
1870
|
+
description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.",
|
1298
1871
|
examples=[
|
1299
|
-
{
|
1300
|
-
{
|
1301
|
-
{
|
1872
|
+
{"sort_order": "ASC", "sort_field": "CREATED_AT"},
|
1873
|
+
{"key": "{{ config['value'] }}"},
|
1874
|
+
{"sort": {"field": "updated_at", "order": "ascending"}},
|
1302
1875
|
],
|
1303
|
-
title=
|
1876
|
+
title="Request Body JSON Payload",
|
1304
1877
|
)
|
1305
1878
|
request_headers: Optional[Union[str, Dict[str, str]]] = Field(
|
1306
1879
|
None,
|
1307
|
-
description=
|
1308
|
-
examples=[{
|
1309
|
-
title=
|
1880
|
+
description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.",
|
1881
|
+
examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
|
1882
|
+
title="Request Headers",
|
1310
1883
|
)
|
1311
1884
|
request_parameters: Optional[Union[str, Dict[str, str]]] = Field(
|
1312
1885
|
None,
|
1313
|
-
description=
|
1886
|
+
description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
|
1314
1887
|
examples=[
|
1315
|
-
{
|
1888
|
+
{"unit": "day"},
|
1316
1889
|
{
|
1317
|
-
|
1890
|
+
"query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
|
1318
1891
|
},
|
1319
|
-
{
|
1320
|
-
{
|
1892
|
+
{"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
|
1893
|
+
{"sort_by[asc]": "updated_at"},
|
1321
1894
|
],
|
1322
|
-
title=
|
1895
|
+
title="Query Parameters",
|
1323
1896
|
)
|
1324
1897
|
use_cache: Optional[bool] = Field(
|
1325
1898
|
False,
|
1326
|
-
description=
|
1327
|
-
title=
|
1899
|
+
description="Enables stream requests caching. This field is automatically set by the CDK.",
|
1900
|
+
title="Use Cache",
|
1901
|
+
)
|
1902
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1903
|
+
|
1904
|
+
|
1905
|
+
class DynamicSchemaLoader(BaseModel):
|
1906
|
+
type: Literal["DynamicSchemaLoader"]
|
1907
|
+
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
|
1908
|
+
...,
|
1909
|
+
description="Component used to coordinate how records are extracted across stream slices and request pages.",
|
1910
|
+
title="Retriever",
|
1911
|
+
)
|
1912
|
+
schema_transformations: Optional[
|
1913
|
+
List[
|
1914
|
+
Union[
|
1915
|
+
AddFields,
|
1916
|
+
CustomTransformation,
|
1917
|
+
RemoveFields,
|
1918
|
+
KeysToLower,
|
1919
|
+
KeysToSnakeCase,
|
1920
|
+
FlattenFields,
|
1921
|
+
KeysReplace,
|
1922
|
+
]
|
1923
|
+
]
|
1924
|
+
] = Field(
|
1925
|
+
None,
|
1926
|
+
description="A list of transformations to be applied to the schema.",
|
1927
|
+
title="Schema Transformations",
|
1328
1928
|
)
|
1329
|
-
|
1929
|
+
schema_type_identifier: SchemaTypeIdentifier
|
1930
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1330
1931
|
|
1331
1932
|
|
1332
1933
|
class ParentStreamConfig(BaseModel):
|
1333
|
-
type: Literal[
|
1934
|
+
type: Literal["ParentStreamConfig"]
|
1334
1935
|
parent_key: str = Field(
|
1335
1936
|
...,
|
1336
|
-
description=
|
1337
|
-
examples=[
|
1338
|
-
title=
|
1937
|
+
description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.",
|
1938
|
+
examples=["id", "{{ config['parent_record_id'] }}"],
|
1939
|
+
title="Parent Key",
|
1339
1940
|
)
|
1340
1941
|
stream: DeclarativeStream = Field(
|
1341
|
-
..., description=
|
1942
|
+
..., description="Reference to the parent stream.", title="Parent Stream"
|
1342
1943
|
)
|
1343
1944
|
partition_field: str = Field(
|
1344
1945
|
...,
|
1345
|
-
description=
|
1346
|
-
examples=[
|
1347
|
-
title=
|
1946
|
+
description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.",
|
1947
|
+
examples=["parent_id", "{{ config['parent_partition_field'] }}"],
|
1948
|
+
title="Current Parent Key Value Identifier",
|
1348
1949
|
)
|
1349
1950
|
request_option: Optional[RequestOption] = Field(
|
1350
1951
|
None,
|
1351
|
-
description=
|
1352
|
-
title=
|
1952
|
+
description="A request option describing where the parent key value should be injected into and under what field name if applicable.",
|
1953
|
+
title="Request Option",
|
1954
|
+
)
|
1955
|
+
incremental_dependency: Optional[bool] = Field(
|
1956
|
+
False,
|
1957
|
+
description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.",
|
1958
|
+
title="Incremental Dependency",
|
1959
|
+
)
|
1960
|
+
extra_fields: Optional[List[List[str]]] = Field(
|
1961
|
+
None,
|
1962
|
+
description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.",
|
1963
|
+
title="Extra Fields",
|
1353
1964
|
)
|
1354
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1965
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1355
1966
|
|
1356
1967
|
|
1357
1968
|
class SimpleRetriever(BaseModel):
|
1358
|
-
type: Literal[
|
1969
|
+
type: Literal["SimpleRetriever"]
|
1359
1970
|
record_selector: RecordSelector = Field(
|
1360
1971
|
...,
|
1361
|
-
description=
|
1972
|
+
description="Component that describes how to extract records from a HTTP response.",
|
1362
1973
|
)
|
1363
1974
|
requester: Union[CustomRequester, HttpRequester] = Field(
|
1364
1975
|
...,
|
1365
|
-
description=
|
1976
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API.",
|
1366
1977
|
)
|
1367
1978
|
paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
|
1368
1979
|
None,
|
@@ -1370,40 +1981,163 @@ class SimpleRetriever(BaseModel):
|
|
1370
1981
|
)
|
1371
1982
|
ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field(
|
1372
1983
|
False,
|
1373
|
-
description=
|
1984
|
+
description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.",
|
1985
|
+
)
|
1986
|
+
partition_router: Optional[
|
1987
|
+
Union[
|
1988
|
+
CustomPartitionRouter,
|
1989
|
+
ListPartitionRouter,
|
1990
|
+
SubstreamPartitionRouter,
|
1991
|
+
List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
|
1992
|
+
]
|
1993
|
+
] = Field(
|
1994
|
+
[],
|
1995
|
+
description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
|
1996
|
+
title="Partition Router",
|
1997
|
+
)
|
1998
|
+
decoder: Optional[
|
1999
|
+
Union[
|
2000
|
+
CustomDecoder,
|
2001
|
+
JsonDecoder,
|
2002
|
+
JsonlDecoder,
|
2003
|
+
IterableDecoder,
|
2004
|
+
XmlDecoder,
|
2005
|
+
GzipJsonDecoder,
|
2006
|
+
CompositeRawDecoder,
|
2007
|
+
]
|
2008
|
+
] = Field(
|
2009
|
+
None,
|
2010
|
+
description="Component decoding the response so records can be extracted.",
|
2011
|
+
title="Decoder",
|
2012
|
+
)
|
2013
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
2014
|
+
|
2015
|
+
|
2016
|
+
class AsyncRetriever(BaseModel):
|
2017
|
+
type: Literal["AsyncRetriever"]
|
2018
|
+
record_selector: RecordSelector = Field(
|
2019
|
+
...,
|
2020
|
+
description="Component that describes how to extract records from a HTTP response.",
|
2021
|
+
)
|
2022
|
+
status_mapping: AsyncJobStatusMap = Field(
|
2023
|
+
..., description="Async Job Status to Airbyte CDK Async Job Status mapping."
|
2024
|
+
)
|
2025
|
+
status_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
|
2026
|
+
..., description="Responsible for fetching the actual status of the async job."
|
2027
|
+
)
|
2028
|
+
urls_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
|
2029
|
+
...,
|
2030
|
+
description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.",
|
2031
|
+
)
|
2032
|
+
download_extractor: Optional[
|
2033
|
+
Union[CustomRecordExtractor, DpathExtractor, ResponseToFileExtractor]
|
2034
|
+
] = Field(None, description="Responsible for fetching the records from provided urls.")
|
2035
|
+
creation_requester: Union[CustomRequester, HttpRequester] = Field(
|
2036
|
+
...,
|
2037
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.",
|
2038
|
+
)
|
2039
|
+
polling_requester: Union[CustomRequester, HttpRequester] = Field(
|
2040
|
+
...,
|
2041
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.",
|
2042
|
+
)
|
2043
|
+
download_requester: Union[CustomRequester, HttpRequester] = Field(
|
2044
|
+
...,
|
2045
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.",
|
2046
|
+
)
|
2047
|
+
download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
|
2048
|
+
None,
|
2049
|
+
description="Paginator component that describes how to navigate through the API's pages during download.",
|
2050
|
+
)
|
2051
|
+
abort_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
|
2052
|
+
None,
|
2053
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.",
|
2054
|
+
)
|
2055
|
+
delete_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
|
2056
|
+
None,
|
2057
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.",
|
1374
2058
|
)
|
1375
2059
|
partition_router: Optional[
|
1376
2060
|
Union[
|
1377
2061
|
CustomPartitionRouter,
|
1378
2062
|
ListPartitionRouter,
|
1379
2063
|
SubstreamPartitionRouter,
|
1380
|
-
List[
|
1381
|
-
Union[
|
1382
|
-
CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
|
1383
|
-
]
|
1384
|
-
],
|
2064
|
+
List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
|
1385
2065
|
]
|
1386
2066
|
] = Field(
|
1387
2067
|
[],
|
1388
|
-
description=
|
1389
|
-
title=
|
2068
|
+
description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
|
2069
|
+
title="Partition Router",
|
2070
|
+
)
|
2071
|
+
decoder: Optional[
|
2072
|
+
Union[
|
2073
|
+
CustomDecoder,
|
2074
|
+
JsonDecoder,
|
2075
|
+
JsonlDecoder,
|
2076
|
+
IterableDecoder,
|
2077
|
+
XmlDecoder,
|
2078
|
+
GzipJsonDecoder,
|
2079
|
+
]
|
2080
|
+
] = Field(
|
2081
|
+
None,
|
2082
|
+
description="Component decoding the response so records can be extracted.",
|
2083
|
+
title="Decoder",
|
2084
|
+
)
|
2085
|
+
download_decoder: Optional[
|
2086
|
+
Union[
|
2087
|
+
CustomDecoder,
|
2088
|
+
JsonDecoder,
|
2089
|
+
JsonlDecoder,
|
2090
|
+
IterableDecoder,
|
2091
|
+
XmlDecoder,
|
2092
|
+
GzipJsonDecoder,
|
2093
|
+
]
|
2094
|
+
] = Field(
|
2095
|
+
None,
|
2096
|
+
description="Component decoding the download response so records can be extracted.",
|
2097
|
+
title="Download Decoder",
|
1390
2098
|
)
|
1391
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
2099
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1392
2100
|
|
1393
2101
|
|
1394
2102
|
class SubstreamPartitionRouter(BaseModel):
|
1395
|
-
type: Literal[
|
2103
|
+
type: Literal["SubstreamPartitionRouter"]
|
1396
2104
|
parent_stream_configs: List[ParentStreamConfig] = Field(
|
1397
2105
|
...,
|
1398
|
-
description=
|
1399
|
-
title=
|
2106
|
+
description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.",
|
2107
|
+
title="Parent Stream Configs",
|
2108
|
+
)
|
2109
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
2110
|
+
|
2111
|
+
|
2112
|
+
class HttpComponentsResolver(BaseModel):
|
2113
|
+
type: Literal["HttpComponentsResolver"]
|
2114
|
+
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
|
2115
|
+
...,
|
2116
|
+
description="Component used to coordinate how records are extracted across stream slices and request pages.",
|
2117
|
+
title="Retriever",
|
2118
|
+
)
|
2119
|
+
components_mapping: List[ComponentMappingDefinition]
|
2120
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
2121
|
+
|
2122
|
+
|
2123
|
+
class DynamicDeclarativeStream(BaseModel):
|
2124
|
+
type: Literal["DynamicDeclarativeStream"]
|
2125
|
+
stream_template: DeclarativeStream = Field(
|
2126
|
+
..., description="Reference to the stream template.", title="Stream Template"
|
2127
|
+
)
|
2128
|
+
components_resolver: Union[HttpComponentsResolver, ConfigComponentsResolver] = Field(
|
2129
|
+
...,
|
2130
|
+
description="Component resolve and populates stream templates with components values.",
|
2131
|
+
title="Components Resolver",
|
1400
2132
|
)
|
1401
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1402
2133
|
|
1403
2134
|
|
1404
2135
|
CompositeErrorHandler.update_forward_refs()
|
1405
|
-
|
2136
|
+
DeclarativeSource1.update_forward_refs()
|
2137
|
+
DeclarativeSource2.update_forward_refs()
|
1406
2138
|
SelectiveAuthenticator.update_forward_refs()
|
1407
2139
|
DeclarativeStream.update_forward_refs()
|
1408
2140
|
SessionTokenAuthenticator.update_forward_refs()
|
2141
|
+
DynamicSchemaLoader.update_forward_refs()
|
1409
2142
|
SimpleRetriever.update_forward_refs()
|
2143
|
+
AsyncRetriever.update_forward_refs()
|