airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev1__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- airbyte_cdk/__init__.py +355 -6
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +29 -10
- airbyte_cdk/connector.py +24 -24
- airbyte_cdk/connector_builder/README.md +53 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
- airbyte_cdk/connector_builder/main.py +45 -13
- airbyte_cdk/connector_builder/message_grouper.py +189 -50
- airbyte_cdk/connector_builder/models.py +3 -2
- airbyte_cdk/destinations/__init__.py +4 -3
- airbyte_cdk/destinations/destination.py +54 -20
- airbyte_cdk/destinations/vector_db_based/README.md +37 -0
- airbyte_cdk/destinations/vector_db_based/config.py +40 -17
- airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
- airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
- airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
- airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
- airbyte_cdk/entrypoint.py +153 -44
- airbyte_cdk/exception_handler.py +21 -3
- airbyte_cdk/logger.py +30 -44
- airbyte_cdk/models/__init__.py +13 -2
- airbyte_cdk/models/airbyte_protocol.py +86 -1
- airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
- airbyte_cdk/models/file_transfer_record_message.py +13 -0
- airbyte_cdk/models/well_known_types.py +1 -1
- airbyte_cdk/sources/__init__.py +5 -1
- airbyte_cdk/sources/abstract_source.py +125 -79
- airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
- airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
- airbyte_cdk/sources/config.py +3 -2
- airbyte_cdk/sources/connector_state_manager.py +49 -83
- airbyte_cdk/sources/declarative/async_job/job.py +52 -0
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
- airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
- airbyte_cdk/sources/declarative/async_job/status.py +24 -0
- airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
- airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
- airbyte_cdk/sources/declarative/auth/token.py +28 -10
- airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
- airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
- airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
- airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1185 -85
- airbyte_cdk/sources/declarative/declarative_source.py +5 -2
- airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
- airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
- airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
- airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
- airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
- airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
- airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
- airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
- airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
- airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +340 -0
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
- airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +174 -74
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
- airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
- airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
- airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
- airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
- airbyte_cdk/sources/declarative/models/__init__.py +1 -1
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1319 -603
- airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1759 -225
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
- airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
- airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
- airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
- airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
- airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
- airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
- airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
- airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
- airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
- airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
- airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
- airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
- airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
- airbyte_cdk/sources/declarative/spec/spec.py +12 -5
- airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
- airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
- airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
- airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
- airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
- airbyte_cdk/sources/declarative/types.py +19 -110
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
- airbyte_cdk/sources/embedded/base_integration.py +16 -5
- airbyte_cdk/sources/embedded/catalog.py +16 -4
- airbyte_cdk/sources/embedded/runner.py +19 -3
- airbyte_cdk/sources/embedded/tools.py +5 -2
- airbyte_cdk/sources/file_based/README.md +152 -0
- airbyte_cdk/sources/file_based/__init__.py +24 -0
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
- airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
- airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
- airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
- airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
- airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
- airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
- airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
- airbyte_cdk/sources/file_based/exceptions.py +18 -15
- airbyte_cdk/sources/file_based/file_based_source.py +140 -33
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
- airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
- airbyte_cdk/sources/file_based/remote_file.py +1 -1
- airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
- airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
- airbyte_cdk/sources/http_logger.py +8 -3
- airbyte_cdk/sources/message/__init__.py +7 -1
- airbyte_cdk/sources/message/repository.py +18 -4
- airbyte_cdk/sources/source.py +42 -38
- airbyte_cdk/sources/streams/__init__.py +2 -2
- airbyte_cdk/sources/streams/availability_strategy.py +54 -3
- airbyte_cdk/sources/streams/call_rate.py +64 -21
- airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
- airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
- airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
- airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
- airbyte_cdk/sources/streams/concurrent/README.md +7 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
- airbyte_cdk/sources/streams/concurrent/cursor.py +313 -48
- airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
- airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
- airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
- airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
- airbyte_cdk/sources/streams/core.py +412 -87
- airbyte_cdk/sources/streams/http/__init__.py +2 -1
- airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
- airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
- airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
- airbyte_cdk/sources/streams/http/exceptions.py +27 -7
- airbyte_cdk/sources/streams/http/http.py +369 -246
- airbyte_cdk/sources/streams/http/http_client.py +531 -0
- airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
- airbyte_cdk/sources/types.py +154 -0
- airbyte_cdk/sources/utils/record_helper.py +36 -21
- airbyte_cdk/sources/utils/schema_helpers.py +13 -6
- airbyte_cdk/sources/utils/slice_logger.py +4 -1
- airbyte_cdk/sources/utils/transform.py +54 -20
- airbyte_cdk/sql/_util/hashing.py +34 -0
- airbyte_cdk/sql/_util/name_normalizers.py +92 -0
- airbyte_cdk/sql/constants.py +32 -0
- airbyte_cdk/sql/exceptions.py +235 -0
- airbyte_cdk/sql/secrets.py +123 -0
- airbyte_cdk/sql/shared/__init__.py +15 -0
- airbyte_cdk/sql/shared/catalog_providers.py +145 -0
- airbyte_cdk/sql/shared/sql_processor.py +786 -0
- airbyte_cdk/sql/types.py +160 -0
- airbyte_cdk/test/catalog_builder.py +70 -18
- airbyte_cdk/test/entrypoint_wrapper.py +117 -42
- airbyte_cdk/test/mock_http/__init__.py +1 -1
- airbyte_cdk/test/mock_http/matcher.py +6 -0
- airbyte_cdk/test/mock_http/mocker.py +57 -10
- airbyte_cdk/test/mock_http/request.py +19 -3
- airbyte_cdk/test/mock_http/response.py +3 -1
- airbyte_cdk/test/mock_http/response_builder.py +32 -16
- airbyte_cdk/test/state_builder.py +18 -10
- airbyte_cdk/test/utils/__init__.py +1 -0
- airbyte_cdk/test/utils/data.py +24 -0
- airbyte_cdk/test/utils/http_mocking.py +16 -0
- airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
- airbyte_cdk/test/utils/reading.py +26 -0
- airbyte_cdk/utils/__init__.py +2 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
- airbyte_cdk/utils/analytics_message.py +10 -2
- airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
- airbyte_cdk/utils/event_timing.py +10 -10
- airbyte_cdk/utils/mapping_helpers.py +3 -1
- airbyte_cdk/utils/message_utils.py +20 -11
- airbyte_cdk/utils/print_buffer.py +75 -0
- airbyte_cdk/utils/schema_inferrer.py +198 -28
- airbyte_cdk/utils/slice_hasher.py +30 -0
- airbyte_cdk/utils/spec_schema_transformations.py +6 -3
- airbyte_cdk/utils/stream_status_utils.py +8 -1
- airbyte_cdk/utils/traced_exception.py +61 -21
- airbyte_cdk-6.17.1.dev1.dist-info/METADATA +109 -0
- airbyte_cdk-6.17.1.dev1.dist-info/RECORD +350 -0
- {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev1.dist-info}/WHEEL +1 -2
- airbyte_cdk-6.17.1.dev1.dist-info/entry_points.txt +3 -0
- airbyte_cdk/sources/declarative/create_partial.py +0 -92
- airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
- airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
- airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
- airbyte_cdk/sources/deprecated/base_source.py +0 -94
- airbyte_cdk/sources/deprecated/client.py +0 -99
- airbyte_cdk/sources/singer/__init__.py +0 -8
- airbyte_cdk/sources/singer/singer_helpers.py +0 -304
- airbyte_cdk/sources/singer/source.py +0 -186
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
- airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
- airbyte_cdk/sources/streams/http/auth/core.py +0 -29
- airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
- airbyte_cdk/sources/streams/http/auth/token.py +0 -47
- airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
- airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
- airbyte_cdk/sources/utils/schema_models.py +0 -84
- airbyte_cdk-0.72.1.dist-info/METADATA +0 -243
- airbyte_cdk-0.72.1.dist-info/RECORD +0 -466
- airbyte_cdk-0.72.1.dist-info/top_level.txt +0 -3
- source_declarative_manifest/main.py +0 -29
- unit_tests/connector_builder/__init__.py +0 -3
- unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
- unit_tests/connector_builder/test_message_grouper.py +0 -713
- unit_tests/connector_builder/utils.py +0 -27
- unit_tests/destinations/test_destination.py +0 -243
- unit_tests/singer/test_singer_helpers.py +0 -56
- unit_tests/singer/test_singer_source.py +0 -112
- unit_tests/sources/__init__.py +0 -0
- unit_tests/sources/concurrent_source/__init__.py +0 -3
- unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
- unit_tests/sources/declarative/__init__.py +0 -3
- unit_tests/sources/declarative/auth/__init__.py +0 -3
- unit_tests/sources/declarative/auth/test_oauth.py +0 -331
- unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
- unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
- unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
- unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
- unit_tests/sources/declarative/checks/__init__.py +0 -3
- unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
- unit_tests/sources/declarative/decoders/__init__.py +0 -0
- unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
- unit_tests/sources/declarative/external_component.py +0 -13
- unit_tests/sources/declarative/extractors/__init__.py +0 -3
- unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
- unit_tests/sources/declarative/incremental/__init__.py +0 -0
- unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
- unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
- unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
- unit_tests/sources/declarative/interpolation/__init__.py +0 -3
- unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
- unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
- unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
- unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
- unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
- unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
- unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
- unit_tests/sources/declarative/parsers/__init__.py +0 -3
- unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
- unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
- unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1847
- unit_tests/sources/declarative/parsers/testing_components.py +0 -36
- unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
- unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
- unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
- unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
- unit_tests/sources/declarative/requesters/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
- unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
- unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
- unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
- unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
- unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
- unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
- unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
- unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
- unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
- unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
- unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
- unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
- unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
- unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
- unit_tests/sources/declarative/retrievers/__init__.py +0 -3
- unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
- unit_tests/sources/declarative/schema/__init__.py +0 -6
- unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
- unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
- unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
- unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
- unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
- unit_tests/sources/declarative/states/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
- unit_tests/sources/declarative/test_create_partial.py +0 -83
- unit_tests/sources/declarative/test_declarative_stream.py +0 -103
- unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
- unit_tests/sources/declarative/test_types.py +0 -39
- unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
- unit_tests/sources/file_based/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
- unit_tests/sources/file_based/config/__init__.py +0 -0
- unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
- unit_tests/sources/file_based/config/test_csv_format.py +0 -34
- unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
- unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
- unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
- unit_tests/sources/file_based/file_types/__init__.py +0 -0
- unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
- unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
- unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
- unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
- unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
- unit_tests/sources/file_based/helpers.py +0 -70
- unit_tests/sources/file_based/in_memory_files_source.py +0 -211
- unit_tests/sources/file_based/scenarios/__init__.py +0 -0
- unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
- unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
- unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
- unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
- unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
- unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
- unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
- unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
- unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
- unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
- unit_tests/sources/file_based/stream/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
- unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
- unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
- unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
- unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
- unit_tests/sources/file_based/test_scenarios.py +0 -253
- unit_tests/sources/file_based/test_schema_helpers.py +0 -346
- unit_tests/sources/fixtures/__init__.py +0 -3
- unit_tests/sources/fixtures/source_test_fixture.py +0 -153
- unit_tests/sources/message/__init__.py +0 -0
- unit_tests/sources/message/test_repository.py +0 -153
- unit_tests/sources/streams/__init__.py +0 -0
- unit_tests/sources/streams/concurrent/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
- unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
- unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
- unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
- unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
- unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
- unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
- unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
- unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
- unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
- unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
- unit_tests/sources/streams/http/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/test_auth.py +0 -173
- unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
- unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
- unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
- unit_tests/sources/streams/http/test_http.py +0 -635
- unit_tests/sources/streams/test_availability_strategy.py +0 -70
- unit_tests/sources/streams/test_call_rate.py +0 -300
- unit_tests/sources/streams/test_stream_read.py +0 -405
- unit_tests/sources/streams/test_streams_core.py +0 -184
- unit_tests/sources/test_abstract_source.py +0 -1442
- unit_tests/sources/test_concurrent_source.py +0 -112
- unit_tests/sources/test_config.py +0 -92
- unit_tests/sources/test_connector_state_manager.py +0 -482
- unit_tests/sources/test_http_logger.py +0 -252
- unit_tests/sources/test_integration_source.py +0 -86
- unit_tests/sources/test_source.py +0 -684
- unit_tests/sources/test_source_read.py +0 -460
- unit_tests/test/__init__.py +0 -0
- unit_tests/test/mock_http/__init__.py +0 -0
- unit_tests/test/mock_http/test_matcher.py +0 -53
- unit_tests/test/mock_http/test_mocker.py +0 -214
- unit_tests/test/mock_http/test_request.py +0 -117
- unit_tests/test/mock_http/test_response_builder.py +0 -177
- unit_tests/test/test_entrypoint_wrapper.py +0 -240
- unit_tests/utils/__init__.py +0 -0
- unit_tests/utils/test_datetime_format_inferrer.py +0 -60
- unit_tests/utils/test_mapping_helpers.py +0 -54
- unit_tests/utils/test_message_utils.py +0 -91
- unit_tests/utils/test_rate_limiting.py +0 -26
- unit_tests/utils/test_schema_inferrer.py +0 -202
- unit_tests/utils/test_secret_utils.py +0 -135
- unit_tests/utils/test_stream_status_utils.py +0 -61
- unit_tests/utils/test_traced_exception.py +0 -107
- /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
- {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
- {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
- {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
- {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev1.dist-info}/LICENSE.txt +0 -0
@@ -7,8 +7,12 @@ version: 1.0.0
|
|
7
7
|
required:
|
8
8
|
- type
|
9
9
|
- check
|
10
|
-
- streams
|
11
10
|
- version
|
11
|
+
anyOf:
|
12
|
+
- required:
|
13
|
+
- streams
|
14
|
+
- required:
|
15
|
+
- dynamic_streams
|
12
16
|
properties:
|
13
17
|
type:
|
14
18
|
type: string
|
@@ -19,18 +23,28 @@ properties:
|
|
19
23
|
type: array
|
20
24
|
items:
|
21
25
|
"$ref": "#/definitions/DeclarativeStream"
|
26
|
+
dynamic_streams:
|
27
|
+
type: array
|
28
|
+
items:
|
29
|
+
"$ref": "#/definitions/DynamicDeclarativeStream"
|
22
30
|
version:
|
23
31
|
type: string
|
32
|
+
description: The version of the Airbyte CDK used to build and test the source.
|
24
33
|
schemas:
|
25
34
|
"$ref": "#/definitions/Schemas"
|
26
35
|
definitions:
|
27
36
|
type: object
|
28
37
|
spec:
|
29
38
|
"$ref": "#/definitions/Spec"
|
39
|
+
concurrency_level:
|
40
|
+
"$ref": "#/definitions/ConcurrencyLevel"
|
30
41
|
metadata:
|
31
42
|
type: object
|
32
43
|
description: For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.
|
33
44
|
additionalProperties: true
|
45
|
+
description:
|
46
|
+
type: string
|
47
|
+
description: A description of the connector. It will be presented on the Source documentation page.
|
34
48
|
additionalProperties: false
|
35
49
|
definitions:
|
36
50
|
AddedFieldDefinition:
|
@@ -257,6 +271,7 @@ definitions:
|
|
257
271
|
- "$ref": "#/definitions/BearerAuthenticator"
|
258
272
|
- "$ref": "#/definitions/CustomAuthenticator"
|
259
273
|
- "$ref": "#/definitions/OAuthAuthenticator"
|
274
|
+
- "$ref": "#/definitions/JwtAuthenticator"
|
260
275
|
- "$ref": "#/definitions/NoAuth"
|
261
276
|
- "$ref": "#/definitions/SessionTokenAuthenticator"
|
262
277
|
- "$ref": "#/definitions/LegacySessionTokenAuthenticator"
|
@@ -264,6 +279,7 @@ definitions:
|
|
264
279
|
- authenticators:
|
265
280
|
token: "#/definitions/ApiKeyAuthenticator"
|
266
281
|
oauth: "#/definitions/OAuthAuthenticator"
|
282
|
+
jwt: "#/definitions/JwtAuthenticator"
|
267
283
|
$parameters:
|
268
284
|
type: object
|
269
285
|
additionalProperties: true
|
@@ -309,6 +325,37 @@ definitions:
|
|
309
325
|
$parameters:
|
310
326
|
type: object
|
311
327
|
additionalProperties: true
|
328
|
+
ConcurrencyLevel:
|
329
|
+
title: Concurrency Level
|
330
|
+
description: Defines the amount of parallelization for the streams that are being synced. The factor of parallelization is how many partitions or streams are synced at the same time. For example, with a concurrency_level of 10, ten streams or partitions of data will processed at the same time. Note that a value of 1 could create deadlock if a stream has a very high number of partitions.
|
331
|
+
type: object
|
332
|
+
required:
|
333
|
+
- default_concurrency
|
334
|
+
properties:
|
335
|
+
type:
|
336
|
+
type: string
|
337
|
+
enum: [ConcurrencyLevel]
|
338
|
+
default_concurrency:
|
339
|
+
title: Default Concurrency
|
340
|
+
description: The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.
|
341
|
+
anyOf:
|
342
|
+
- type: integer
|
343
|
+
- type: string
|
344
|
+
interpolation_context:
|
345
|
+
- config
|
346
|
+
examples:
|
347
|
+
- 10
|
348
|
+
- "{{ config['num_workers'] or 10 }}"
|
349
|
+
max_concurrency:
|
350
|
+
title: Max Concurrency
|
351
|
+
description: The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.
|
352
|
+
type: integer
|
353
|
+
examples:
|
354
|
+
- 20
|
355
|
+
- 100
|
356
|
+
$parameters:
|
357
|
+
type: object
|
358
|
+
additionalProperties: true
|
312
359
|
ConstantBackoffStrategy:
|
313
360
|
title: Constant Backoff
|
314
361
|
description: Backoff strategy with a constant backoff interval.
|
@@ -353,11 +400,12 @@ definitions:
|
|
353
400
|
interpolation_context:
|
354
401
|
- config
|
355
402
|
- headers
|
356
|
-
-
|
403
|
+
- last_page_size
|
404
|
+
- last_record
|
357
405
|
- response
|
358
406
|
examples:
|
359
407
|
- "{{ headers.link.next.cursor }}"
|
360
|
-
- "{{
|
408
|
+
- "{{ last_record['key'] }}"
|
361
409
|
- "{{ response['nextPage'] }}"
|
362
410
|
page_size:
|
363
411
|
title: Page Size
|
@@ -372,15 +420,11 @@ definitions:
|
|
372
420
|
interpolation_context:
|
373
421
|
- config
|
374
422
|
- headers
|
375
|
-
-
|
423
|
+
- last_record
|
376
424
|
- response
|
377
425
|
examples:
|
378
426
|
- "{{ response.data.has_more is false }}"
|
379
427
|
- "{{ 'next' not in headers['link'] }}"
|
380
|
-
decoder:
|
381
|
-
title: Decoder
|
382
|
-
description: Component decoding the response so records can be extracted.
|
383
|
-
"$ref": "#/definitions/JsonDecoder"
|
384
428
|
$parameters:
|
385
429
|
type: object
|
386
430
|
additionalProperties: true
|
@@ -623,6 +667,49 @@ definitions:
|
|
623
667
|
$parameters:
|
624
668
|
type: object
|
625
669
|
additionalProperties: true
|
670
|
+
CustomSchemaNormalization:
|
671
|
+
title: Custom Schema Normalization
|
672
|
+
description: Schema normalization component whose behavior is derived from a custom code implementation of the connector.
|
673
|
+
type: object
|
674
|
+
additionalProperties: true
|
675
|
+
required:
|
676
|
+
- type
|
677
|
+
- class_name
|
678
|
+
properties:
|
679
|
+
type:
|
680
|
+
type: string
|
681
|
+
enum: [ CustomSchemaNormalization ]
|
682
|
+
class_name:
|
683
|
+
title: Class Name
|
684
|
+
description: Fully-qualified name of the class that will be implementing the custom normalization. The format is `source_<name>.<package>.<class_name>`.
|
685
|
+
type: string
|
686
|
+
additionalProperties: true
|
687
|
+
examples:
|
688
|
+
- "source_amazon_seller_partner.components.LedgerDetailedViewReportsTypeTransformer"
|
689
|
+
$parameters:
|
690
|
+
type: object
|
691
|
+
additionalProperties: true
|
692
|
+
CustomStateMigration:
|
693
|
+
title: Custom State Migration
|
694
|
+
description: Apply a custom transformation on the input state.
|
695
|
+
type: object
|
696
|
+
additionalProperties: true
|
697
|
+
required:
|
698
|
+
- type
|
699
|
+
- class_name
|
700
|
+
properties:
|
701
|
+
type:
|
702
|
+
type: string
|
703
|
+
enum: [CustomStateMigration]
|
704
|
+
class_name:
|
705
|
+
title: Class Name
|
706
|
+
description: Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.
|
707
|
+
type: string
|
708
|
+
examples:
|
709
|
+
- "source_railz.components.MyCustomStateMigration"
|
710
|
+
$parameters:
|
711
|
+
type: object
|
712
|
+
additionalProperties: true
|
626
713
|
CustomTransformation:
|
627
714
|
title: Custom Transformation
|
628
715
|
description: Transformation component whose behavior is derived from a custom code implementation of the connector.
|
@@ -644,6 +731,29 @@ definitions:
|
|
644
731
|
$parameters:
|
645
732
|
type: object
|
646
733
|
additionalProperties: true
|
734
|
+
LegacyToPerPartitionStateMigration:
|
735
|
+
title: Legacy To Per-partition-state Migration
|
736
|
+
description:
|
737
|
+
'Transforms the input state for per-partitioned streams from the legacy format to the low-code format.
|
738
|
+
The cursor field and partition ID fields are automatically extracted from the stream''s DatetimebasedCursor and SubstreamPartitionRouter.
|
739
|
+
|
740
|
+
Example input state:
|
741
|
+
{
|
742
|
+
"13506132": {
|
743
|
+
"last_changed": "2022-12-27T08:34:39+00:00"
|
744
|
+
}
|
745
|
+
Example output state:
|
746
|
+
{
|
747
|
+
"partition": {"id": "13506132"},
|
748
|
+
"cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}
|
749
|
+
}
|
750
|
+
'
|
751
|
+
type: object
|
752
|
+
additionalProperties: true
|
753
|
+
properties:
|
754
|
+
type:
|
755
|
+
type: string
|
756
|
+
enum: [LegacyToPerPartitionStateMigration]
|
647
757
|
DatetimeBasedCursor:
|
648
758
|
title: Datetime Based Cursor
|
649
759
|
description: Cursor to provide incremental capabilities over datetime.
|
@@ -671,6 +781,7 @@ definitions:
|
|
671
781
|
description: |
|
672
782
|
The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
|
673
783
|
* **%s**: Epoch unix timestamp - `1686218963`
|
784
|
+
* **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`
|
674
785
|
* **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`
|
675
786
|
* **%a**: Weekday (abbreviated) - `Sun`
|
676
787
|
* **%A**: Weekday (full) - `Sunday`
|
@@ -704,6 +815,7 @@ definitions:
|
|
704
815
|
- "%Y-%m-%d"
|
705
816
|
- "%s"
|
706
817
|
- "%ms"
|
818
|
+
- "%s_as_float"
|
707
819
|
start_datetime:
|
708
820
|
title: Start Datetime
|
709
821
|
description: The datetime that determines the earliest record that should be synced.
|
@@ -753,6 +865,20 @@ definitions:
|
|
753
865
|
title: Whether the target API is formatted as a data feed
|
754
866
|
description: A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.
|
755
867
|
type: boolean
|
868
|
+
is_client_side_incremental:
|
869
|
+
title: Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)
|
870
|
+
description: If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.
|
871
|
+
type: boolean
|
872
|
+
is_compare_strictly:
|
873
|
+
title: Whether to skip requests if the start time equals the end time
|
874
|
+
description: Set to True if the target API does not accept queries where the start time equal the end time.
|
875
|
+
type: boolean
|
876
|
+
default: False
|
877
|
+
global_substream_cursor:
|
878
|
+
title: Whether to store cursor as one value instead of per partition
|
879
|
+
description: This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).
|
880
|
+
type: boolean
|
881
|
+
default: false
|
756
882
|
lookback_window:
|
757
883
|
title: Lookback Window
|
758
884
|
description: Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.
|
@@ -788,6 +914,127 @@ definitions:
|
|
788
914
|
$parameters:
|
789
915
|
type: object
|
790
916
|
additionalProperties: true
|
917
|
+
JwtAuthenticator:
|
918
|
+
title: JWT Authenticator
|
919
|
+
description: Authenticator for requests using JWT authentication flow.
|
920
|
+
type: object
|
921
|
+
required:
|
922
|
+
- type
|
923
|
+
- secret_key
|
924
|
+
- algorithm
|
925
|
+
properties:
|
926
|
+
type:
|
927
|
+
type: string
|
928
|
+
enum: [JwtAuthenticator]
|
929
|
+
secret_key:
|
930
|
+
type: string
|
931
|
+
description: Secret used to sign the JSON web token.
|
932
|
+
examples:
|
933
|
+
- "{{ config['secret_key'] }}"
|
934
|
+
base64_encode_secret_key:
|
935
|
+
type: boolean
|
936
|
+
description: When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.
|
937
|
+
default: False
|
938
|
+
algorithm:
|
939
|
+
type: string
|
940
|
+
description: Algorithm used to sign the JSON web token.
|
941
|
+
enum:
|
942
|
+
[
|
943
|
+
"HS256",
|
944
|
+
"HS384",
|
945
|
+
"HS512",
|
946
|
+
"ES256",
|
947
|
+
"ES256K",
|
948
|
+
"ES384",
|
949
|
+
"ES512",
|
950
|
+
"RS256",
|
951
|
+
"RS384",
|
952
|
+
"RS512",
|
953
|
+
"PS256",
|
954
|
+
"PS384",
|
955
|
+
"PS512",
|
956
|
+
"EdDSA",
|
957
|
+
]
|
958
|
+
examples:
|
959
|
+
- ES256
|
960
|
+
- HS256
|
961
|
+
- RS256
|
962
|
+
- "{{ config['algorithm'] }}"
|
963
|
+
token_duration:
|
964
|
+
type: integer
|
965
|
+
title: Token Duration
|
966
|
+
description: The amount of time in seconds a JWT token can be valid after being issued.
|
967
|
+
default: 1200
|
968
|
+
examples:
|
969
|
+
- 1200
|
970
|
+
- 3600
|
971
|
+
header_prefix:
|
972
|
+
type: string
|
973
|
+
title: Header Prefix
|
974
|
+
description: The prefix to be used within the Authentication header.
|
975
|
+
examples:
|
976
|
+
- "Bearer"
|
977
|
+
- "Basic"
|
978
|
+
jwt_headers:
|
979
|
+
type: object
|
980
|
+
title: JWT Headers
|
981
|
+
description: JWT headers used when signing JSON web token.
|
982
|
+
additionalProperties: false
|
983
|
+
properties:
|
984
|
+
kid:
|
985
|
+
type: string
|
986
|
+
title: Key Identifier
|
987
|
+
description: Private key ID for user account.
|
988
|
+
examples:
|
989
|
+
- "{{ config['kid'] }}"
|
990
|
+
typ:
|
991
|
+
type: string
|
992
|
+
title: Type
|
993
|
+
description: The media type of the complete JWT.
|
994
|
+
default: JWT
|
995
|
+
examples:
|
996
|
+
- JWT
|
997
|
+
cty:
|
998
|
+
type: string
|
999
|
+
title: Content Type
|
1000
|
+
description: Content type of JWT header.
|
1001
|
+
examples:
|
1002
|
+
- JWT
|
1003
|
+
additional_jwt_headers:
|
1004
|
+
type: object
|
1005
|
+
title: Additional JWT Headers
|
1006
|
+
description: Additional headers to be included with the JWT headers object.
|
1007
|
+
additionalProperties: true
|
1008
|
+
jwt_payload:
|
1009
|
+
type: object
|
1010
|
+
title: JWT Payload
|
1011
|
+
description: JWT Payload used when signing JSON web token.
|
1012
|
+
additionalProperties: false
|
1013
|
+
properties:
|
1014
|
+
iss:
|
1015
|
+
type: string
|
1016
|
+
title: Issuer
|
1017
|
+
description: The user/principal that issued the JWT. Commonly a value unique to the user.
|
1018
|
+
examples:
|
1019
|
+
- "{{ config['iss'] }}"
|
1020
|
+
sub:
|
1021
|
+
type: string
|
1022
|
+
title: Subject
|
1023
|
+
description: The subject of the JWT. Commonly defined by the API.
|
1024
|
+
aud:
|
1025
|
+
type: string
|
1026
|
+
title: Audience
|
1027
|
+
description: The recipient that the JWT is intended for. Commonly defined by the API.
|
1028
|
+
examples:
|
1029
|
+
- "appstoreconnect-v1"
|
1030
|
+
additional_jwt_payload:
|
1031
|
+
type: object
|
1032
|
+
title: Additional JWT Payload Properties
|
1033
|
+
description: Additional properties to be added to the JWT payload.
|
1034
|
+
additionalProperties: true
|
1035
|
+
$parameters:
|
1036
|
+
type: object
|
1037
|
+
additionalProperties: true
|
791
1038
|
OAuthAuthenticator:
|
792
1039
|
title: OAuth2
|
793
1040
|
description: Authenticator for requests using OAuth 2.0 authorization flow.
|
@@ -796,7 +1043,6 @@ definitions:
|
|
796
1043
|
- type
|
797
1044
|
- client_id
|
798
1045
|
- client_secret
|
799
|
-
- token_refresh_endpoint
|
800
1046
|
properties:
|
801
1047
|
type:
|
802
1048
|
type: string
|
@@ -835,6 +1081,12 @@ definitions:
|
|
835
1081
|
default: "access_token"
|
836
1082
|
examples:
|
837
1083
|
- access_token
|
1084
|
+
access_token_value:
|
1085
|
+
title: Access Token Value
|
1086
|
+
description: The value of the access_token to bypass the token refreshing using `refresh_token`.
|
1087
|
+
type: string
|
1088
|
+
examples:
|
1089
|
+
- secret_access_token_value
|
838
1090
|
expires_in_name:
|
839
1091
|
title: Token Expiry Property Name
|
840
1092
|
description: The name of the property which contains the expiry date in the response from the token refresh endpoint.
|
@@ -968,6 +1220,7 @@ definitions:
|
|
968
1220
|
title: Retriever
|
969
1221
|
description: Component used to coordinate how records are extracted across stream slices and request pages.
|
970
1222
|
anyOf:
|
1223
|
+
- "$ref": "#/definitions/AsyncRetriever"
|
971
1224
|
- "$ref": "#/definitions/CustomRetriever"
|
972
1225
|
- "$ref": "#/definitions/SimpleRetriever"
|
973
1226
|
incremental_sync:
|
@@ -992,6 +1245,7 @@ definitions:
|
|
992
1245
|
title: Schema Loader
|
993
1246
|
description: Component used to retrieve the schema for the current stream.
|
994
1247
|
anyOf:
|
1248
|
+
- "$ref": "#/definitions/DynamicSchemaLoader"
|
995
1249
|
- "$ref": "#/definitions/InlineSchemaLoader"
|
996
1250
|
- "$ref": "#/definitions/JsonFileSchemaLoader"
|
997
1251
|
- "$ref": "#/definitions/CustomSchemaLoader"
|
@@ -1006,6 +1260,19 @@ definitions:
|
|
1006
1260
|
- "$ref": "#/definitions/AddFields"
|
1007
1261
|
- "$ref": "#/definitions/CustomTransformation"
|
1008
1262
|
- "$ref": "#/definitions/RemoveFields"
|
1263
|
+
- "$ref": "#/definitions/KeysToLower"
|
1264
|
+
- "$ref": "#/definitions/KeysToSnakeCase"
|
1265
|
+
- "$ref": "#/definitions/FlattenFields"
|
1266
|
+
- "$ref": "#/definitions/KeysReplace"
|
1267
|
+
state_migrations:
|
1268
|
+
title: State Migrations
|
1269
|
+
description: Array of state migrations to be applied on the input state
|
1270
|
+
type: array
|
1271
|
+
items:
|
1272
|
+
anyOf:
|
1273
|
+
- "$ref": "#/definitions/LegacyToPerPartitionStateMigration"
|
1274
|
+
- "$ref": "#/definitions/CustomStateMigration"
|
1275
|
+
default: []
|
1009
1276
|
$parameters:
|
1010
1277
|
type: object
|
1011
1278
|
additional_properties: true
|
@@ -1067,10 +1334,6 @@ definitions:
|
|
1067
1334
|
- "$ref": "#/definitions/CustomPaginationStrategy"
|
1068
1335
|
- "$ref": "#/definitions/OffsetIncrement"
|
1069
1336
|
- "$ref": "#/definitions/PageIncrement"
|
1070
|
-
decoder:
|
1071
|
-
title: Decoder
|
1072
|
-
description: Component decoding the response so records can be extracted.
|
1073
|
-
"$ref": "#/definitions/JsonDecoder"
|
1074
1337
|
page_size_option:
|
1075
1338
|
"$ref": "#/definitions/RequestOption"
|
1076
1339
|
page_token_option:
|
@@ -1097,17 +1360,26 @@ definitions:
|
|
1097
1360
|
type: array
|
1098
1361
|
items:
|
1099
1362
|
- type: string
|
1100
|
-
|
1363
|
+
interpolation_context:
|
1101
1364
|
- config
|
1102
1365
|
examples:
|
1103
1366
|
- ["data"]
|
1104
1367
|
- ["data", "records"]
|
1105
1368
|
- ["data", "{{ parameters.name }}"]
|
1106
1369
|
- ["data", "*", "record"]
|
1107
|
-
|
1108
|
-
|
1109
|
-
|
1110
|
-
|
1370
|
+
$parameters:
|
1371
|
+
type: object
|
1372
|
+
additionalProperties: true
|
1373
|
+
ResponseToFileExtractor:
|
1374
|
+
title: CSV To File Extractor
|
1375
|
+
description: A record extractor designed for handling large responses that may exceed memory limits (to prevent OOM issues). It downloads a CSV file to disk, reads the data from disk, and deletes the file once it has been fully processed.
|
1376
|
+
type: object
|
1377
|
+
required:
|
1378
|
+
- type
|
1379
|
+
properties:
|
1380
|
+
type:
|
1381
|
+
type: string
|
1382
|
+
enum: [ResponseToFileExtractor]
|
1111
1383
|
$parameters:
|
1112
1384
|
type: object
|
1113
1385
|
additionalProperties: true
|
@@ -1182,6 +1454,12 @@ definitions:
|
|
1182
1454
|
anyOf:
|
1183
1455
|
- "$ref": "#/definitions/SessionTokenRequestApiKeyAuthenticator"
|
1184
1456
|
- "$ref": "#/definitions/SessionTokenRequestBearerAuthenticator"
|
1457
|
+
decoder:
|
1458
|
+
title: Decoder
|
1459
|
+
description: Component used to decode the response.
|
1460
|
+
anyOf:
|
1461
|
+
- "$ref": "#/definitions/JsonDecoder"
|
1462
|
+
- "$ref": "#/definitions/XmlDecoder"
|
1185
1463
|
$parameters:
|
1186
1464
|
type: object
|
1187
1465
|
additionalProperties: true
|
@@ -1257,6 +1535,7 @@ definitions:
|
|
1257
1535
|
- "$ref": "#/definitions/BearerAuthenticator"
|
1258
1536
|
- "$ref": "#/definitions/CustomAuthenticator"
|
1259
1537
|
- "$ref": "#/definitions/OAuthAuthenticator"
|
1538
|
+
- "$ref": "#/definitions/JwtAuthenticator"
|
1260
1539
|
- "$ref": "#/definitions/NoAuth"
|
1261
1540
|
- "$ref": "#/definitions/SessionTokenAuthenticator"
|
1262
1541
|
- "$ref": "#/definitions/LegacySessionTokenAuthenticator"
|
@@ -1367,7 +1646,6 @@ definitions:
|
|
1367
1646
|
type: object
|
1368
1647
|
required:
|
1369
1648
|
- type
|
1370
|
-
- action
|
1371
1649
|
properties:
|
1372
1650
|
type:
|
1373
1651
|
type: string
|
@@ -1381,11 +1659,25 @@ definitions:
|
|
1381
1659
|
- FAIL
|
1382
1660
|
- RETRY
|
1383
1661
|
- IGNORE
|
1662
|
+
- RATE_LIMITED
|
1384
1663
|
examples:
|
1385
1664
|
- SUCCESS
|
1386
1665
|
- FAIL
|
1387
1666
|
- RETRY
|
1388
1667
|
- IGNORE
|
1668
|
+
- RATE_LIMITED
|
1669
|
+
failure_type:
|
1670
|
+
title: Failure Type
|
1671
|
+
description: Failure type of traced exception if a response matches the filter.
|
1672
|
+
type: string
|
1673
|
+
enum:
|
1674
|
+
- system_error
|
1675
|
+
- config_error
|
1676
|
+
- transient_error
|
1677
|
+
examples:
|
1678
|
+
- system_error
|
1679
|
+
- config_error
|
1680
|
+
- transient_error
|
1389
1681
|
error_message:
|
1390
1682
|
title: Error Message
|
1391
1683
|
description: Error Message to display if the response matches the filter.
|
@@ -1423,6 +1715,105 @@ definitions:
|
|
1423
1715
|
$parameters:
|
1424
1716
|
type: object
|
1425
1717
|
additionalProperties: true
|
1718
|
+
TypesMap:
|
1719
|
+
title: Types Map
|
1720
|
+
description: (This component is experimental. Use at your own risk.) Represents a mapping between a current type and its corresponding target type.
|
1721
|
+
type: object
|
1722
|
+
required:
|
1723
|
+
- target_type
|
1724
|
+
- current_type
|
1725
|
+
properties:
|
1726
|
+
target_type:
|
1727
|
+
anyOf:
|
1728
|
+
- type: string
|
1729
|
+
- type: array
|
1730
|
+
items:
|
1731
|
+
type: string
|
1732
|
+
current_type:
|
1733
|
+
anyOf:
|
1734
|
+
- type: string
|
1735
|
+
- type: array
|
1736
|
+
items:
|
1737
|
+
type: string
|
1738
|
+
SchemaTypeIdentifier:
|
1739
|
+
title: Schema Type Identifier
|
1740
|
+
description: (This component is experimental. Use at your own risk.) Identifies schema details for dynamic schema extraction and processing.
|
1741
|
+
type: object
|
1742
|
+
required:
|
1743
|
+
- key_pointer
|
1744
|
+
properties:
|
1745
|
+
type:
|
1746
|
+
type: string
|
1747
|
+
enum: [SchemaTypeIdentifier]
|
1748
|
+
schema_pointer:
|
1749
|
+
title: Schema Path
|
1750
|
+
description: List of nested fields defining the schema field path to extract. Defaults to [].
|
1751
|
+
type: array
|
1752
|
+
default: []
|
1753
|
+
items:
|
1754
|
+
- type: string
|
1755
|
+
interpolation_context:
|
1756
|
+
- config
|
1757
|
+
key_pointer:
|
1758
|
+
title: Key Path
|
1759
|
+
description: List of potentially nested fields describing the full path of the field key to extract.
|
1760
|
+
type: array
|
1761
|
+
items:
|
1762
|
+
- type: string
|
1763
|
+
interpolation_context:
|
1764
|
+
- config
|
1765
|
+
type_pointer:
|
1766
|
+
title: Type Path
|
1767
|
+
description: List of potentially nested fields describing the full path of the field type to extract.
|
1768
|
+
type: array
|
1769
|
+
items:
|
1770
|
+
- type: string
|
1771
|
+
interpolation_context:
|
1772
|
+
- config
|
1773
|
+
types_mapping:
|
1774
|
+
type: array
|
1775
|
+
items:
|
1776
|
+
- "$ref": "#/definitions/TypesMap"
|
1777
|
+
$parameters:
|
1778
|
+
type: object
|
1779
|
+
additionalProperties: true
|
1780
|
+
DynamicSchemaLoader:
|
1781
|
+
title: Dynamic Schema Loader
|
1782
|
+
description: (This component is experimental. Use at your own risk.) Loads a schema by extracting data from retrieved records.
|
1783
|
+
type: object
|
1784
|
+
required:
|
1785
|
+
- type
|
1786
|
+
- retriever
|
1787
|
+
- schema_type_identifier
|
1788
|
+
properties:
|
1789
|
+
type:
|
1790
|
+
type: string
|
1791
|
+
enum: [DynamicSchemaLoader]
|
1792
|
+
retriever:
|
1793
|
+
title: Retriever
|
1794
|
+
description: Component used to coordinate how records are extracted across stream slices and request pages.
|
1795
|
+
anyOf:
|
1796
|
+
- "$ref": "#/definitions/AsyncRetriever"
|
1797
|
+
- "$ref": "#/definitions/CustomRetriever"
|
1798
|
+
- "$ref": "#/definitions/SimpleRetriever"
|
1799
|
+
schema_transformations:
|
1800
|
+
title: Schema Transformations
|
1801
|
+
description: A list of transformations to be applied to the schema.
|
1802
|
+
type: array
|
1803
|
+
items:
|
1804
|
+
anyOf:
|
1805
|
+
- "$ref": "#/definitions/AddFields"
|
1806
|
+
- "$ref": "#/definitions/CustomTransformation"
|
1807
|
+
- "$ref": "#/definitions/RemoveFields"
|
1808
|
+
- "$ref": "#/definitions/KeysToLower"
|
1809
|
+
- "$ref": "#/definitions/KeysToSnakeCase"
|
1810
|
+
- "$ref": "#/definitions/FlattenFields"
|
1811
|
+
- "$ref": "#/definitions/KeysReplace"
|
1812
|
+
schema_type_identifier:
|
1813
|
+
"$ref": "#/definitions/SchemaTypeIdentifier"
|
1814
|
+
$parameters:
|
1815
|
+
type: object
|
1816
|
+
additionalProperties: true
|
1426
1817
|
InlineSchemaLoader:
|
1427
1818
|
title: Inline Schema Loader
|
1428
1819
|
description: Loads a schema that is defined directly in the manifest file.
|
@@ -1467,65 +1858,221 @@ definitions:
|
|
1467
1858
|
type:
|
1468
1859
|
type: string
|
1469
1860
|
enum: [JsonDecoder]
|
1470
|
-
|
1471
|
-
title:
|
1472
|
-
description:
|
1861
|
+
JsonlDecoder:
|
1862
|
+
title: JSONL Decoder
|
1863
|
+
description: Use this if the response consists of JSON objects separated by new lines (`\n`) in JSONL format.
|
1473
1864
|
type: object
|
1474
1865
|
required:
|
1475
1866
|
- type
|
1476
|
-
- cursor_field
|
1477
|
-
- values
|
1478
1867
|
properties:
|
1479
1868
|
type:
|
1480
1869
|
type: string
|
1481
|
-
enum: [
|
1482
|
-
|
1483
|
-
|
1484
|
-
|
1870
|
+
enum: [JsonlDecoder]
|
1871
|
+
KeysToLower:
|
1872
|
+
title: Keys to Lower Case
|
1873
|
+
description: A transformation that renames all keys to lower case.
|
1874
|
+
type: object
|
1875
|
+
required:
|
1876
|
+
- type
|
1877
|
+
properties:
|
1878
|
+
type:
|
1485
1879
|
type: string
|
1486
|
-
|
1487
|
-
- config
|
1488
|
-
examples:
|
1489
|
-
- "section"
|
1490
|
-
- "{{ config['section_key'] }}"
|
1491
|
-
values:
|
1492
|
-
title: Partition Values
|
1493
|
-
description: The list of attributes being iterated over and used as input for the requests made to the source API.
|
1494
|
-
anyOf:
|
1495
|
-
- type: string
|
1496
|
-
- type: array
|
1497
|
-
items:
|
1498
|
-
type: string
|
1499
|
-
interpolation_context:
|
1500
|
-
- config
|
1501
|
-
examples:
|
1502
|
-
- ["section_a", "section_b", "section_c"]
|
1503
|
-
- "{{ config['sections'] }}"
|
1504
|
-
request_option:
|
1505
|
-
title: Inject Partition Value Into Outgoing HTTP Request
|
1506
|
-
description: A request option describing where the list value should be injected into and under what field name if applicable.
|
1507
|
-
"$ref": "#/definitions/RequestOption"
|
1880
|
+
enum: [KeysToLower]
|
1508
1881
|
$parameters:
|
1509
1882
|
type: object
|
1510
1883
|
additionalProperties: true
|
1511
|
-
|
1512
|
-
title:
|
1513
|
-
description:
|
1884
|
+
KeysToSnakeCase:
|
1885
|
+
title: Key to Snake Case
|
1886
|
+
description: A transformation that renames all keys to snake case.
|
1514
1887
|
type: object
|
1515
1888
|
required:
|
1516
1889
|
- type
|
1517
|
-
- datetime
|
1518
1890
|
properties:
|
1519
1891
|
type:
|
1520
1892
|
type: string
|
1521
|
-
enum: [
|
1522
|
-
|
1523
|
-
|
1524
|
-
|
1893
|
+
enum: [KeysToSnakeCase]
|
1894
|
+
$parameters:
|
1895
|
+
type: object
|
1896
|
+
additionalProperties: true
|
1897
|
+
FlattenFields:
|
1898
|
+
title: Flatten Fields
|
1899
|
+
description: A transformation that flatten record to single level format.
|
1900
|
+
type: object
|
1901
|
+
required:
|
1902
|
+
- type
|
1903
|
+
properties:
|
1904
|
+
type:
|
1905
|
+
type: string
|
1906
|
+
enum: [FlattenFields]
|
1907
|
+
flatten_lists:
|
1908
|
+
title: Flatten Lists
|
1909
|
+
description: Whether to flatten lists or leave it as is. Default is True.
|
1910
|
+
type: boolean
|
1911
|
+
default: true
|
1912
|
+
$parameters:
|
1913
|
+
type: object
|
1914
|
+
additionalProperties: true
|
1915
|
+
KeysReplace:
|
1916
|
+
title: Keys Replace
|
1917
|
+
description: A transformation that replaces symbols in keys.
|
1918
|
+
type: object
|
1919
|
+
required:
|
1920
|
+
- type
|
1921
|
+
- old
|
1922
|
+
- new
|
1923
|
+
properties:
|
1924
|
+
type:
|
1925
|
+
type: string
|
1926
|
+
enum: [KeysReplace]
|
1927
|
+
old:
|
1525
1928
|
type: string
|
1929
|
+
title: Old value
|
1930
|
+
description: Old value to replace.
|
1931
|
+
examples:
|
1932
|
+
- " "
|
1933
|
+
- "{{ record.id }}"
|
1934
|
+
- "{{ config['id'] }}"
|
1935
|
+
- "{{ stream_slice['id'] }}"
|
1526
1936
|
interpolation_context:
|
1527
1937
|
- config
|
1528
|
-
|
1938
|
+
- record
|
1939
|
+
- stream_state
|
1940
|
+
- stream_slice
|
1941
|
+
new:
|
1942
|
+
type: string
|
1943
|
+
title: New value
|
1944
|
+
description: New value to set.
|
1945
|
+
examples:
|
1946
|
+
- "_"
|
1947
|
+
- "{{ record.id }}"
|
1948
|
+
- "{{ config['id'] }}"
|
1949
|
+
- "{{ stream_slice['id'] }}"
|
1950
|
+
interpolation_context:
|
1951
|
+
- config
|
1952
|
+
- record
|
1953
|
+
- stream_state
|
1954
|
+
- stream_slice
|
1955
|
+
$parameters:
|
1956
|
+
type: object
|
1957
|
+
additionalProperties: true
|
1958
|
+
IterableDecoder:
|
1959
|
+
title: Iterable Decoder
|
1960
|
+
description: Use this if the response consists of strings separated by new lines (`\n`). The Decoder will wrap each row into a JSON object with the `record` key.
|
1961
|
+
type: object
|
1962
|
+
required:
|
1963
|
+
- type
|
1964
|
+
properties:
|
1965
|
+
type:
|
1966
|
+
type: string
|
1967
|
+
enum: [IterableDecoder]
|
1968
|
+
XmlDecoder:
|
1969
|
+
title: XML Decoder
|
1970
|
+
description: Use this if the response is XML.
|
1971
|
+
type: object
|
1972
|
+
required:
|
1973
|
+
- type
|
1974
|
+
properties:
|
1975
|
+
type:
|
1976
|
+
type: string
|
1977
|
+
enum: [XmlDecoder]
|
1978
|
+
CustomDecoder:
|
1979
|
+
title: Custom Decoder
|
1980
|
+
description: Use this to implement custom decoder logic.
|
1981
|
+
type: object
|
1982
|
+
additionalProperties: true
|
1983
|
+
required:
|
1984
|
+
- type
|
1985
|
+
- class_name
|
1986
|
+
properties:
|
1987
|
+
type:
|
1988
|
+
type: string
|
1989
|
+
enum: [CustomDecoder]
|
1990
|
+
class_name:
|
1991
|
+
title: Class Name
|
1992
|
+
description: Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.
|
1993
|
+
type: string
|
1994
|
+
additionalProperties: true
|
1995
|
+
examples:
|
1996
|
+
- "source_amazon_ads.components.GzipJsonlDecoder"
|
1997
|
+
$parameters:
|
1998
|
+
type: object
|
1999
|
+
additionalProperties: true
|
2000
|
+
GzipJsonDecoder:
|
2001
|
+
title: GzipJson Decoder
|
2002
|
+
description: Use this if the response is Gzip compressed Json.
|
2003
|
+
type: object
|
2004
|
+
additionalProperties: true
|
2005
|
+
required:
|
2006
|
+
- type
|
2007
|
+
properties:
|
2008
|
+
type:
|
2009
|
+
type: string
|
2010
|
+
enum: [GzipJsonDecoder]
|
2011
|
+
encoding:
|
2012
|
+
type: string
|
2013
|
+
default: utf-8
|
2014
|
+
$parameters:
|
2015
|
+
type: object
|
2016
|
+
additionalProperties: true
|
2017
|
+
ListPartitionRouter:
|
2018
|
+
title: List Partition Router
|
2019
|
+
description: A Partition router that specifies a list of attributes where each attribute describes a portion of the complete data set for a stream. During a sync, each value is iterated over and can be used as input to outbound API requests.
|
2020
|
+
type: object
|
2021
|
+
required:
|
2022
|
+
- type
|
2023
|
+
- cursor_field
|
2024
|
+
- values
|
2025
|
+
properties:
|
2026
|
+
type:
|
2027
|
+
type: string
|
2028
|
+
enum: [ListPartitionRouter]
|
2029
|
+
cursor_field:
|
2030
|
+
title: Current Partition Value Identifier
|
2031
|
+
description: While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition['my_key'] }}" where "my_key" is the value of the cursor_field.
|
2032
|
+
type: string
|
2033
|
+
interpolation_context:
|
2034
|
+
- config
|
2035
|
+
examples:
|
2036
|
+
- "section"
|
2037
|
+
- "{{ config['section_key'] }}"
|
2038
|
+
values:
|
2039
|
+
title: Partition Values
|
2040
|
+
description: The list of attributes being iterated over and used as input for the requests made to the source API.
|
2041
|
+
anyOf:
|
2042
|
+
- type: string
|
2043
|
+
- type: array
|
2044
|
+
items:
|
2045
|
+
type: string
|
2046
|
+
interpolation_context:
|
2047
|
+
- config
|
2048
|
+
examples:
|
2049
|
+
- ["section_a", "section_b", "section_c"]
|
2050
|
+
- "{{ config['sections'] }}"
|
2051
|
+
request_option:
|
2052
|
+
title: Inject Partition Value Into Outgoing HTTP Request
|
2053
|
+
description: A request option describing where the list value should be injected into and under what field name if applicable.
|
2054
|
+
"$ref": "#/definitions/RequestOption"
|
2055
|
+
$parameters:
|
2056
|
+
type: object
|
2057
|
+
additionalProperties: true
|
2058
|
+
MinMaxDatetime:
|
2059
|
+
title: Min-Max Datetime
|
2060
|
+
description: Compares the provided date against optional minimum or maximum times. The max_datetime serves as the ceiling and will be returned when datetime exceeds it. The min_datetime serves as the floor.
|
2061
|
+
type: object
|
2062
|
+
required:
|
2063
|
+
- type
|
2064
|
+
- datetime
|
2065
|
+
properties:
|
2066
|
+
type:
|
2067
|
+
type: string
|
2068
|
+
enum: [MinMaxDatetime]
|
2069
|
+
datetime:
|
2070
|
+
title: Datetime
|
2071
|
+
description: Datetime value.
|
2072
|
+
type: string
|
2073
|
+
interpolation_context:
|
2074
|
+
- config
|
2075
|
+
examples:
|
1529
2076
|
- 2021-01-01
|
1530
2077
|
- 2021-01-01T00:00:00Z
|
1531
2078
|
- "{{ config['start_time'] }}"
|
@@ -1534,6 +2081,7 @@ definitions:
|
|
1534
2081
|
description: |
|
1535
2082
|
Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:
|
1536
2083
|
* **%s**: Epoch unix timestamp - `1686218963`
|
2084
|
+
* **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`
|
1537
2085
|
* **%ms**: Epoch unix timestamp - `1686218963123`
|
1538
2086
|
* **%a**: Weekday (abbreviated) - `Sun`
|
1539
2087
|
* **%A**: Weekday (full) - `Sunday`
|
@@ -1649,6 +2197,170 @@ definitions:
|
|
1649
2197
|
- app_id:
|
1650
2198
|
type: string
|
1651
2199
|
path_in_connector_config: ["info", "app_id"]
|
2200
|
+
oauth_connector_input_specification:
|
2201
|
+
title: DeclarativeOAuth Connector Specification
|
2202
|
+
description: |-
|
2203
|
+
The DeclarativeOAuth specific blob.
|
2204
|
+
Pertains to the fields defined by the connector relating to the OAuth flow.
|
2205
|
+
|
2206
|
+
Interpolation capabilities:
|
2207
|
+
- The variables placeholders are declared as `{my_var}`.
|
2208
|
+
- The nested resolution variables like `{{my_nested_var}}` is allowed as well.
|
2209
|
+
|
2210
|
+
- The allowed interpolation context is:
|
2211
|
+
+ base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}
|
2212
|
+
+ base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}
|
2213
|
+
+ urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}
|
2214
|
+
+ urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}
|
2215
|
+
+ codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}
|
2216
|
+
|
2217
|
+
Examples:
|
2218
|
+
- The TikTok Marketing DeclarativeOAuth spec:
|
2219
|
+
{
|
2220
|
+
"oauth_connector_input_specification": {
|
2221
|
+
"type": "object",
|
2222
|
+
"additionalProperties": false,
|
2223
|
+
"properties": {
|
2224
|
+
"consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",
|
2225
|
+
"access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",
|
2226
|
+
"access_token_params": {
|
2227
|
+
"{auth_code_key}": "{{auth_code_key}}",
|
2228
|
+
"{client_id_key}": "{{client_id_key}}",
|
2229
|
+
"{client_secret_key}": "{{client_secret_key}}"
|
2230
|
+
},
|
2231
|
+
"access_token_headers": {
|
2232
|
+
"Content-Type": "application/json",
|
2233
|
+
"Accept": "application/json"
|
2234
|
+
},
|
2235
|
+
"extract_output": ["data.access_token"],
|
2236
|
+
"client_id_key": "app_id",
|
2237
|
+
"client_secret_key": "secret",
|
2238
|
+
"auth_code_key": "auth_code"
|
2239
|
+
}
|
2240
|
+
}
|
2241
|
+
}
|
2242
|
+
type: object
|
2243
|
+
additionalProperties: true
|
2244
|
+
required:
|
2245
|
+
- consent_url
|
2246
|
+
- access_token_url
|
2247
|
+
- extract_output
|
2248
|
+
properties:
|
2249
|
+
consent_url:
|
2250
|
+
title: Consent URL
|
2251
|
+
type: string
|
2252
|
+
description: |-
|
2253
|
+
The DeclarativeOAuth Specific string URL string template to initiate the authentication.
|
2254
|
+
The placeholders are replaced during the processing to provide neccessary values.
|
2255
|
+
examples:
|
2256
|
+
- https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}
|
2257
|
+
- https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}
|
2258
|
+
scope:
|
2259
|
+
title: Scopes
|
2260
|
+
type: string
|
2261
|
+
description: |-
|
2262
|
+
The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.
|
2263
|
+
examples:
|
2264
|
+
- user:read user:read_orders workspaces:read
|
2265
|
+
access_token_url:
|
2266
|
+
title: Access Token URL
|
2267
|
+
type: string
|
2268
|
+
description: |-
|
2269
|
+
The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.
|
2270
|
+
The placeholders are replaced during the processing to provide neccessary values.
|
2271
|
+
examples:
|
2272
|
+
- https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}
|
2273
|
+
access_token_headers:
|
2274
|
+
title: Access Token Headers
|
2275
|
+
type: object
|
2276
|
+
additionalProperties: true
|
2277
|
+
description: |-
|
2278
|
+
The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
|
2279
|
+
examples:
|
2280
|
+
- {
|
2281
|
+
"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}",
|
2282
|
+
}
|
2283
|
+
access_token_params:
|
2284
|
+
title: Access Token Query Params (Json Encoded)
|
2285
|
+
type: object
|
2286
|
+
additionalProperties: true
|
2287
|
+
description: |-
|
2288
|
+
The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
|
2289
|
+
When this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.
|
2290
|
+
examples:
|
2291
|
+
- {
|
2292
|
+
"{auth_code_key}": "{{auth_code_key}}",
|
2293
|
+
"{client_id_key}": "{{client_id_key}}",
|
2294
|
+
"{client_secret_key}": "{{client_secret_key}}",
|
2295
|
+
}
|
2296
|
+
extract_output:
|
2297
|
+
title: Extract Output
|
2298
|
+
type: array
|
2299
|
+
items:
|
2300
|
+
type: string
|
2301
|
+
description: |-
|
2302
|
+
The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.
|
2303
|
+
examples:
|
2304
|
+
- ["access_token", "refresh_token", "other_field"]
|
2305
|
+
state:
|
2306
|
+
title: Configurable State Query Param
|
2307
|
+
type: object
|
2308
|
+
additionalProperties: true
|
2309
|
+
required:
|
2310
|
+
- min
|
2311
|
+
- max
|
2312
|
+
description: |-
|
2313
|
+
The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,
|
2314
|
+
including length and complexity.
|
2315
|
+
properties:
|
2316
|
+
min:
|
2317
|
+
type: integer
|
2318
|
+
max:
|
2319
|
+
type: integer
|
2320
|
+
examples:
|
2321
|
+
- { "min": 7, "max": 128 }
|
2322
|
+
client_id_key:
|
2323
|
+
title: Client ID Key Override
|
2324
|
+
type: string
|
2325
|
+
description: |-
|
2326
|
+
The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.
|
2327
|
+
examples:
|
2328
|
+
- "my_custom_client_id_key_name"
|
2329
|
+
client_secret_key:
|
2330
|
+
title: Client Secret Key Override
|
2331
|
+
type: string
|
2332
|
+
description: |-
|
2333
|
+
The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.
|
2334
|
+
examples:
|
2335
|
+
- "my_custom_client_secret_key_name"
|
2336
|
+
scope_key:
|
2337
|
+
title: Scopes Key Override
|
2338
|
+
type: string
|
2339
|
+
description: |-
|
2340
|
+
The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.
|
2341
|
+
examples:
|
2342
|
+
- "my_custom_scope_key_key_name"
|
2343
|
+
state_key:
|
2344
|
+
title: State Key Override
|
2345
|
+
type: string
|
2346
|
+
description: |-
|
2347
|
+
The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.
|
2348
|
+
examples:
|
2349
|
+
- "my_custom_state_key_key_name"
|
2350
|
+
auth_code_key:
|
2351
|
+
title: Auth Code Key Override
|
2352
|
+
type: string
|
2353
|
+
description: |-
|
2354
|
+
The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.
|
2355
|
+
examples:
|
2356
|
+
- "my_custom_auth_code_key_name"
|
2357
|
+
redirect_uri_key:
|
2358
|
+
title: Redirect URI Key Override
|
2359
|
+
type: string
|
2360
|
+
description: |-
|
2361
|
+
The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.
|
2362
|
+
examples:
|
2363
|
+
- "my_custom_redirect_uri_key_name"
|
1652
2364
|
complete_oauth_output_specification:
|
1653
2365
|
title: "OAuth output specification"
|
1654
2366
|
description: |-
|
@@ -1826,6 +2538,25 @@ definitions:
|
|
1826
2538
|
title: Request Option
|
1827
2539
|
description: A request option describing where the parent key value should be injected into and under what field name if applicable.
|
1828
2540
|
"$ref": "#/definitions/RequestOption"
|
2541
|
+
incremental_dependency:
|
2542
|
+
title: Incremental Dependency
|
2543
|
+
description: Indicates whether the parent stream should be read incrementally based on updates in the child stream.
|
2544
|
+
type: boolean
|
2545
|
+
default: false
|
2546
|
+
extra_fields:
|
2547
|
+
title: Extra Fields
|
2548
|
+
description: Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.
|
2549
|
+
interpolation_context:
|
2550
|
+
- config
|
2551
|
+
type: array
|
2552
|
+
items:
|
2553
|
+
type: array
|
2554
|
+
items:
|
2555
|
+
type: string
|
2556
|
+
description: Defines a field path as an array of strings.
|
2557
|
+
examples:
|
2558
|
+
- ["field1"]
|
2559
|
+
- ["nested", "field2"]
|
1829
2560
|
$parameters:
|
1830
2561
|
type: object
|
1831
2562
|
additionalProperties: true
|
@@ -1896,7 +2627,11 @@ definitions:
|
|
1896
2627
|
- "$ref": "#/definitions/CustomRecordFilter"
|
1897
2628
|
- "$ref": "#/definitions/RecordFilter"
|
1898
2629
|
schema_normalization:
|
1899
|
-
|
2630
|
+
title: Schema Normalization
|
2631
|
+
description: Responsible for normalization according to the schema.
|
2632
|
+
anyOf:
|
2633
|
+
- "$ref": "#/definitions/SchemaNormalization"
|
2634
|
+
- "$ref": "#/definitions/CustomSchemaNormalization"
|
1900
2635
|
default: None
|
1901
2636
|
$parameters:
|
1902
2637
|
type: object
|
@@ -2097,6 +2832,205 @@ definitions:
|
|
2097
2832
|
- "$ref": "#/definitions/CustomPartitionRouter"
|
2098
2833
|
- "$ref": "#/definitions/ListPartitionRouter"
|
2099
2834
|
- "$ref": "#/definitions/SubstreamPartitionRouter"
|
2835
|
+
decoder:
|
2836
|
+
title: Decoder
|
2837
|
+
description: Component decoding the response so records can be extracted.
|
2838
|
+
anyOf:
|
2839
|
+
- "$ref": "#/definitions/CustomDecoder"
|
2840
|
+
- "$ref": "#/definitions/JsonDecoder"
|
2841
|
+
- "$ref": "#/definitions/JsonlDecoder"
|
2842
|
+
- "$ref": "#/definitions/IterableDecoder"
|
2843
|
+
- "$ref": "#/definitions/XmlDecoder"
|
2844
|
+
- "$ref": "#/definitions/GzipJsonDecoder"
|
2845
|
+
- "$ref": "#/definitions/CompositeRawDecoder"
|
2846
|
+
$parameters:
|
2847
|
+
type: object
|
2848
|
+
additionalProperties: true
|
2849
|
+
CompositeRawDecoder:
|
2850
|
+
description: "(This is experimental, use at your own risk)"
|
2851
|
+
type: object
|
2852
|
+
required:
|
2853
|
+
- type
|
2854
|
+
- parser
|
2855
|
+
properties:
|
2856
|
+
type:
|
2857
|
+
type: string
|
2858
|
+
enum: [CompositeRawDecoder]
|
2859
|
+
parser:
|
2860
|
+
anyOf:
|
2861
|
+
- "$ref": "#/definitions/GzipParser"
|
2862
|
+
- "$ref": "#/definitions/JsonLineParser"
|
2863
|
+
- "$ref": "#/definitions/CsvParser"
|
2864
|
+
# PARSERS
|
2865
|
+
GzipParser:
|
2866
|
+
type: object
|
2867
|
+
required:
|
2868
|
+
- type
|
2869
|
+
- inner_parser
|
2870
|
+
properties:
|
2871
|
+
type:
|
2872
|
+
type: string
|
2873
|
+
enum: [GzipParser]
|
2874
|
+
inner_parser:
|
2875
|
+
anyOf:
|
2876
|
+
- "$ref": "#/definitions/JsonLineParser"
|
2877
|
+
- "$ref": "#/definitions/CsvParser"
|
2878
|
+
JsonLineParser:
|
2879
|
+
type: object
|
2880
|
+
required:
|
2881
|
+
- type
|
2882
|
+
properties:
|
2883
|
+
type:
|
2884
|
+
type: string
|
2885
|
+
enum: [JsonLineParser]
|
2886
|
+
encoding:
|
2887
|
+
type: string
|
2888
|
+
default: utf-8
|
2889
|
+
CsvParser:
|
2890
|
+
type: object
|
2891
|
+
required:
|
2892
|
+
- type
|
2893
|
+
properties:
|
2894
|
+
type:
|
2895
|
+
type: string
|
2896
|
+
enum: [CsvParser]
|
2897
|
+
encoding:
|
2898
|
+
type: string
|
2899
|
+
default: utf-8
|
2900
|
+
delimiter:
|
2901
|
+
type: string
|
2902
|
+
default: ","
|
2903
|
+
AsyncJobStatusMap:
|
2904
|
+
description: Matches the api job status to Async Job Status.
|
2905
|
+
type: object
|
2906
|
+
required:
|
2907
|
+
- running
|
2908
|
+
- completed
|
2909
|
+
- failed
|
2910
|
+
- timeout
|
2911
|
+
properties:
|
2912
|
+
type:
|
2913
|
+
type: string
|
2914
|
+
enum: [AsyncJobStatusMap]
|
2915
|
+
running:
|
2916
|
+
type: array
|
2917
|
+
items:
|
2918
|
+
type: string
|
2919
|
+
completed:
|
2920
|
+
type: array
|
2921
|
+
items:
|
2922
|
+
type: string
|
2923
|
+
failed:
|
2924
|
+
type: array
|
2925
|
+
items:
|
2926
|
+
type: string
|
2927
|
+
timeout:
|
2928
|
+
type: array
|
2929
|
+
items:
|
2930
|
+
type: string
|
2931
|
+
AsyncRetriever:
|
2932
|
+
description: "[Experimental - We expect the interface to change shortly and we reserve the right to not consider this a breaking change] Retrieves records by Asynchronously sending requests to fetch records. The retriever acts as an orchestrator between the requester, the record selector, the paginator, and the partition router."
|
2933
|
+
type: object
|
2934
|
+
required:
|
2935
|
+
- type
|
2936
|
+
- record_selector
|
2937
|
+
- status_mapping
|
2938
|
+
- creation_requester
|
2939
|
+
- polling_requester
|
2940
|
+
- download_requester
|
2941
|
+
- status_extractor
|
2942
|
+
- urls_extractor
|
2943
|
+
properties:
|
2944
|
+
type:
|
2945
|
+
type: string
|
2946
|
+
enum: [AsyncRetriever]
|
2947
|
+
record_selector:
|
2948
|
+
description: Component that describes how to extract records from a HTTP response.
|
2949
|
+
"$ref": "#/definitions/RecordSelector"
|
2950
|
+
status_mapping:
|
2951
|
+
description: Async Job Status to Airbyte CDK Async Job Status mapping.
|
2952
|
+
anyOf:
|
2953
|
+
- "$ref": "#/definitions/AsyncJobStatusMap"
|
2954
|
+
status_extractor:
|
2955
|
+
description: Responsible for fetching the actual status of the async job.
|
2956
|
+
anyOf:
|
2957
|
+
- "$ref": "#/definitions/CustomRecordExtractor"
|
2958
|
+
- "$ref": "#/definitions/DpathExtractor"
|
2959
|
+
urls_extractor:
|
2960
|
+
description: Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.
|
2961
|
+
anyOf:
|
2962
|
+
- "$ref": "#/definitions/CustomRecordExtractor"
|
2963
|
+
- "$ref": "#/definitions/DpathExtractor"
|
2964
|
+
download_extractor:
|
2965
|
+
description: Responsible for fetching the records from provided urls.
|
2966
|
+
anyOf:
|
2967
|
+
- "$ref": "#/definitions/CustomRecordExtractor"
|
2968
|
+
- "$ref": "#/definitions/DpathExtractor"
|
2969
|
+
- "$ref": "#/definitions/ResponseToFileExtractor"
|
2970
|
+
creation_requester:
|
2971
|
+
description: Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.
|
2972
|
+
anyOf:
|
2973
|
+
- "$ref": "#/definitions/CustomRequester"
|
2974
|
+
- "$ref": "#/definitions/HttpRequester"
|
2975
|
+
polling_requester:
|
2976
|
+
description: Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.
|
2977
|
+
anyOf:
|
2978
|
+
- "$ref": "#/definitions/CustomRequester"
|
2979
|
+
- "$ref": "#/definitions/HttpRequester"
|
2980
|
+
download_requester:
|
2981
|
+
description: Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.
|
2982
|
+
anyOf:
|
2983
|
+
- "$ref": "#/definitions/CustomRequester"
|
2984
|
+
- "$ref": "#/definitions/HttpRequester"
|
2985
|
+
download_paginator:
|
2986
|
+
description: Paginator component that describes how to navigate through the API's pages during download.
|
2987
|
+
anyOf:
|
2988
|
+
- "$ref": "#/definitions/DefaultPaginator"
|
2989
|
+
- "$ref": "#/definitions/NoPagination"
|
2990
|
+
abort_requester:
|
2991
|
+
description: Requester component that describes how to prepare HTTP requests to send to the source API to abort a job once it is timed out from the source's perspective.
|
2992
|
+
anyOf:
|
2993
|
+
- "$ref": "#/definitions/CustomRequester"
|
2994
|
+
- "$ref": "#/definitions/HttpRequester"
|
2995
|
+
delete_requester:
|
2996
|
+
description: Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.
|
2997
|
+
anyOf:
|
2998
|
+
- "$ref": "#/definitions/CustomRequester"
|
2999
|
+
- "$ref": "#/definitions/HttpRequester"
|
3000
|
+
partition_router:
|
3001
|
+
title: Partition Router
|
3002
|
+
description: PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.
|
3003
|
+
default: []
|
3004
|
+
anyOf:
|
3005
|
+
- "$ref": "#/definitions/CustomPartitionRouter"
|
3006
|
+
- "$ref": "#/definitions/ListPartitionRouter"
|
3007
|
+
- "$ref": "#/definitions/SubstreamPartitionRouter"
|
3008
|
+
- type: array
|
3009
|
+
items:
|
3010
|
+
anyOf:
|
3011
|
+
- "$ref": "#/definitions/CustomPartitionRouter"
|
3012
|
+
- "$ref": "#/definitions/ListPartitionRouter"
|
3013
|
+
- "$ref": "#/definitions/SubstreamPartitionRouter"
|
3014
|
+
decoder:
|
3015
|
+
title: Decoder
|
3016
|
+
description: Component decoding the response so records can be extracted.
|
3017
|
+
anyOf:
|
3018
|
+
- "$ref": "#/definitions/CustomDecoder"
|
3019
|
+
- "$ref": "#/definitions/JsonDecoder"
|
3020
|
+
- "$ref": "#/definitions/JsonlDecoder"
|
3021
|
+
- "$ref": "#/definitions/IterableDecoder"
|
3022
|
+
- "$ref": "#/definitions/XmlDecoder"
|
3023
|
+
- "$ref": "#/definitions/GzipJsonDecoder"
|
3024
|
+
download_decoder:
|
3025
|
+
title: Download Decoder
|
3026
|
+
description: Component decoding the download response so records can be extracted.
|
3027
|
+
anyOf:
|
3028
|
+
- "$ref": "#/definitions/CustomDecoder"
|
3029
|
+
- "$ref": "#/definitions/JsonDecoder"
|
3030
|
+
- "$ref": "#/definitions/JsonlDecoder"
|
3031
|
+
- "$ref": "#/definitions/IterableDecoder"
|
3032
|
+
- "$ref": "#/definitions/XmlDecoder"
|
3033
|
+
- "$ref": "#/definitions/GzipJsonDecoder"
|
2100
3034
|
$parameters:
|
2101
3035
|
type: object
|
2102
3036
|
additionalProperties: true
|
@@ -2180,6 +3114,12 @@ definitions:
|
|
2180
3114
|
type: string
|
2181
3115
|
examples:
|
2182
3116
|
- "([-+]?\\d+)"
|
3117
|
+
max_waiting_time_in_seconds:
|
3118
|
+
title: Max Waiting Time in Seconds
|
3119
|
+
description: Given the value extracted from the header is greater than this value, stop the stream.
|
3120
|
+
type: number
|
3121
|
+
examples:
|
3122
|
+
- 3600
|
2183
3123
|
$parameters:
|
2184
3124
|
type: object
|
2185
3125
|
additionalProperties: true
|
@@ -2224,6 +3164,150 @@ definitions:
|
|
2224
3164
|
$parameters:
|
2225
3165
|
type: object
|
2226
3166
|
additionalProperties: true
|
3167
|
+
ComponentMappingDefinition:
|
3168
|
+
title: Component Mapping Definition
|
3169
|
+
description: (This component is experimental. Use at your own risk.) Specifies a mapping definition to update or add fields in a record or configuration. This allows dynamic mapping of data by interpolating values into the template based on provided contexts.
|
3170
|
+
type: object
|
3171
|
+
required:
|
3172
|
+
- type
|
3173
|
+
- field_path
|
3174
|
+
- value
|
3175
|
+
properties:
|
3176
|
+
type:
|
3177
|
+
type: string
|
3178
|
+
enum: [ComponentMappingDefinition]
|
3179
|
+
field_path:
|
3180
|
+
title: Field Path
|
3181
|
+
description: A list of potentially nested fields indicating the full path where value will be added or updated.
|
3182
|
+
type: array
|
3183
|
+
items:
|
3184
|
+
- type: string
|
3185
|
+
interpolation_context:
|
3186
|
+
- config
|
3187
|
+
- components_values
|
3188
|
+
- stream_slice
|
3189
|
+
- stream_template_config
|
3190
|
+
examples:
|
3191
|
+
- ["data"]
|
3192
|
+
- ["data", "records"]
|
3193
|
+
- ["data", 1, "name"]
|
3194
|
+
- ["data", "{{ components_values.name }}"]
|
3195
|
+
- ["data", "*", "record"]
|
3196
|
+
- ["*", "**", "name"]
|
3197
|
+
value:
|
3198
|
+
title: Value
|
3199
|
+
description: The dynamic or static value to assign to the key. Interpolated values can be used to dynamically determine the value during runtime.
|
3200
|
+
type: string
|
3201
|
+
interpolation_context:
|
3202
|
+
- config
|
3203
|
+
- stream_template_config
|
3204
|
+
- components_values
|
3205
|
+
- stream_slice
|
3206
|
+
examples:
|
3207
|
+
- "{{ components_values['updates'] }}"
|
3208
|
+
- "{{ components_values['MetaData']['LastUpdatedTime'] }}"
|
3209
|
+
- "{{ config['segment_id'] }}"
|
3210
|
+
- "{{ stream_slice['parent_id'] }}"
|
3211
|
+
- "{{ stream_slice['extra_fields']['name'] }}"
|
3212
|
+
value_type:
|
3213
|
+
title: Value Type
|
3214
|
+
description: The expected data type of the value. If omitted, the type will be inferred from the value provided.
|
3215
|
+
"$ref": "#/definitions/ValueType"
|
3216
|
+
$parameters:
|
3217
|
+
type: object
|
3218
|
+
additionalProperties: true
|
3219
|
+
HttpComponentsResolver:
|
3220
|
+
type: object
|
3221
|
+
description: (This component is experimental. Use at your own risk.) Component resolve and populates stream templates with components fetched via an HTTP retriever.
|
3222
|
+
properties:
|
3223
|
+
type:
|
3224
|
+
type: string
|
3225
|
+
enum: [HttpComponentsResolver]
|
3226
|
+
retriever:
|
3227
|
+
title: Retriever
|
3228
|
+
description: Component used to coordinate how records are extracted across stream slices and request pages.
|
3229
|
+
anyOf:
|
3230
|
+
- "$ref": "#/definitions/AsyncRetriever"
|
3231
|
+
- "$ref": "#/definitions/CustomRetriever"
|
3232
|
+
- "$ref": "#/definitions/SimpleRetriever"
|
3233
|
+
components_mapping:
|
3234
|
+
type: array
|
3235
|
+
items:
|
3236
|
+
"$ref": "#/definitions/ComponentMappingDefinition"
|
3237
|
+
$parameters:
|
3238
|
+
type: object
|
3239
|
+
additionalProperties: true
|
3240
|
+
required:
|
3241
|
+
- type
|
3242
|
+
- retriever
|
3243
|
+
- components_mapping
|
3244
|
+
StreamConfig:
|
3245
|
+
title: Stream Config
|
3246
|
+
description: (This component is experimental. Use at your own risk.) Describes how to get streams config from the source config.
|
3247
|
+
type: object
|
3248
|
+
required:
|
3249
|
+
- type
|
3250
|
+
- configs_pointer
|
3251
|
+
properties:
|
3252
|
+
type:
|
3253
|
+
type: string
|
3254
|
+
enum: [StreamConfig]
|
3255
|
+
configs_pointer:
|
3256
|
+
title: Configs Pointer
|
3257
|
+
description: A list of potentially nested fields indicating the full path in source config file where streams configs located.
|
3258
|
+
type: array
|
3259
|
+
items:
|
3260
|
+
- type: string
|
3261
|
+
interpolation_context:
|
3262
|
+
- parameters
|
3263
|
+
examples:
|
3264
|
+
- ["data"]
|
3265
|
+
- ["data", "streams"]
|
3266
|
+
- ["data", "{{ parameters.name }}"]
|
3267
|
+
$parameters:
|
3268
|
+
type: object
|
3269
|
+
additionalProperties: true
|
3270
|
+
ConfigComponentsResolver:
|
3271
|
+
type: object
|
3272
|
+
description: (This component is experimental. Use at your own risk.) Resolves and populates stream templates with components fetched from the source config.
|
3273
|
+
properties:
|
3274
|
+
type:
|
3275
|
+
type: string
|
3276
|
+
enum: [ConfigComponentsResolver]
|
3277
|
+
stream_config:
|
3278
|
+
"$ref": "#/definitions/StreamConfig"
|
3279
|
+
components_mapping:
|
3280
|
+
type: array
|
3281
|
+
items:
|
3282
|
+
"$ref": "#/definitions/ComponentMappingDefinition"
|
3283
|
+
$parameters:
|
3284
|
+
type: object
|
3285
|
+
additionalProperties: true
|
3286
|
+
required:
|
3287
|
+
- type
|
3288
|
+
- stream_config
|
3289
|
+
- components_mapping
|
3290
|
+
DynamicDeclarativeStream:
|
3291
|
+
type: object
|
3292
|
+
description: (This component is experimental. Use at your own risk.) A component that described how will be created declarative streams based on stream template.
|
3293
|
+
properties:
|
3294
|
+
type:
|
3295
|
+
type: string
|
3296
|
+
enum: [DynamicDeclarativeStream]
|
3297
|
+
stream_template:
|
3298
|
+
title: Stream Template
|
3299
|
+
description: Reference to the stream template.
|
3300
|
+
"$ref": "#/definitions/DeclarativeStream"
|
3301
|
+
components_resolver:
|
3302
|
+
title: Components Resolver
|
3303
|
+
description: Component resolve and populates stream templates with components values.
|
3304
|
+
anyOf:
|
3305
|
+
- "$ref": "#/definitions/HttpComponentsResolver"
|
3306
|
+
- "$ref": "#/definitions/ConfigComponentsResolver"
|
3307
|
+
required:
|
3308
|
+
- type
|
3309
|
+
- stream_template
|
3310
|
+
- components_resolver
|
2227
3311
|
interpolation:
|
2228
3312
|
variables:
|
2229
3313
|
- title: config
|
@@ -2253,20 +3337,20 @@ interpolation:
|
|
2253
3337
|
x-ratelimit-limit: "600"
|
2254
3338
|
x-ratelimit-remaining: "598"
|
2255
3339
|
x-ratelimit-reset: "39"
|
2256
|
-
- title:
|
2257
|
-
description:
|
2258
|
-
type:
|
3340
|
+
- title: last_record
|
3341
|
+
description: Last record extracted from the response received from the API.
|
3342
|
+
type: object
|
3343
|
+
examples:
|
3344
|
+
- name: "Test List: 19"
|
3345
|
+
id: 0236d6d2
|
3346
|
+
contact_count: 20
|
3347
|
+
_metadata:
|
3348
|
+
self: https://api.sendgrid.com/v3/marketing/lists/0236d6d2
|
3349
|
+
- title: last_page_size
|
3350
|
+
description: Number of records extracted from the last response received from the API.
|
3351
|
+
type: object
|
2259
3352
|
examples:
|
2260
|
-
-
|
2261
|
-
id: 0236d6d2
|
2262
|
-
contact_count: 20
|
2263
|
-
_metadata:
|
2264
|
-
self: https://api.sendgrid.com/v3/marketing/lists/0236d6d2
|
2265
|
-
- name: List for CI tests, number 30
|
2266
|
-
id: 041ee031
|
2267
|
-
contact_count: 0
|
2268
|
-
_metadata:
|
2269
|
-
self: https://api.sendgrid.com/v3/marketing/lists/041ee031
|
3353
|
+
- 2
|
2270
3354
|
- title: next_page_token
|
2271
3355
|
description: Object describing the token to fetch the next page of records. The object has a single key "next_page_token".
|
2272
3356
|
type: object
|
@@ -2314,21 +3398,21 @@ interpolation:
|
|
2314
3398
|
- created_at: "2020-01-01 00:00:00.000+00:00"
|
2315
3399
|
- updated_at: "2020-01-02 00:00:00.000+00:00"
|
2316
3400
|
macros:
|
2317
|
-
- title:
|
3401
|
+
- title: now_utc
|
2318
3402
|
description: Returns the current date and time in the UTC timezone.
|
2319
3403
|
arguments: {}
|
2320
3404
|
return_type: Datetime
|
2321
3405
|
examples:
|
2322
3406
|
- "'{{ now_utc() }}' -> '2021-09-01 00:00:00+00:00'"
|
2323
3407
|
- "'{{ now_utc().strftime('%Y-%m-%d') }}' -> '2021-09-01'"
|
2324
|
-
- title:
|
3408
|
+
- title: today_utc
|
2325
3409
|
description: Returns the current date in UTC timezone. The output is a date object.
|
2326
3410
|
arguments: {}
|
2327
3411
|
return_type: Date
|
2328
3412
|
examples:
|
2329
3413
|
- "'{{ today_utc() }}' -> '2021-09-01'"
|
2330
3414
|
- "'{{ today_utc().strftime('%Y/%m/%d')}}' -> '2021/09/01'"
|
2331
|
-
- title:
|
3415
|
+
- title: timestamp
|
2332
3416
|
description: Converts a number or a string representing a datetime (formatted as ISO8601) to a timestamp. If the input is a number, it is converted to an int. If no timezone is specified, the string is interpreted as UTC.
|
2333
3417
|
arguments:
|
2334
3418
|
datetime: A string formatted as ISO8601 or an integer representing a unix timestamp
|
@@ -2339,7 +3423,7 @@ interpolation:
|
|
2339
3423
|
- "'{{ timestamp('2022-02-28T00:00:00Z') }}' -> 1646006400"
|
2340
3424
|
- "'{{ timestamp('2022-02-28 00:00:00Z') }}' -> 1646006400"
|
2341
3425
|
- "'{{ timestamp('2022-02-28T00:00:00-08:00') }}' -> 1646035200"
|
2342
|
-
- title:
|
3426
|
+
- title: max
|
2343
3427
|
description: Returns the largest object of a iterable, or or two or more arguments.
|
2344
3428
|
arguments:
|
2345
3429
|
args: iterable or a sequence of two or more arguments
|
@@ -2347,7 +3431,7 @@ interpolation:
|
|
2347
3431
|
examples:
|
2348
3432
|
- "'{{ max(2, 3) }}' -> 3"
|
2349
3433
|
- "'{{ max([2, 3]) }}' -> 3"
|
2350
|
-
- title:
|
3434
|
+
- title: day_delta
|
2351
3435
|
description: Returns the datetime of now() + num_days.
|
2352
3436
|
arguments:
|
2353
3437
|
num_days: The number of days to add to now
|
@@ -2357,8 +3441,8 @@ interpolation:
|
|
2357
3441
|
- "'{{ day_delta(1) }}' -> '2021-09-02T00:00:00.000000+0000'"
|
2358
3442
|
- "'{{ day_delta(-1) }}' -> '2021-08-31:00:00.000000+0000'"
|
2359
3443
|
- "'{{ day_delta(25, format='%Y-%m-%d') }}' -> '2021-09-02'"
|
2360
|
-
- title:
|
2361
|
-
description: Converts an ISO8601
|
3444
|
+
- title: duration
|
3445
|
+
description: Converts an ISO8601 duration to datetime timedelta.
|
2362
3446
|
arguments:
|
2363
3447
|
duration_string: "A string representing an ISO8601 duration. See https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm for more details."
|
2364
3448
|
return_type: datetime.timedelta
|
@@ -2366,17 +3450,19 @@ interpolation:
|
|
2366
3450
|
- "'{{ duration('P1D') }}' -> '1 day, 0:00:00'"
|
2367
3451
|
- "'{{ duration('P6DT23H') }}' -> '6 days, 23:00:00'"
|
2368
3452
|
- "'{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}' -> '2021-08-31T00:00:00Z'"
|
2369
|
-
- title:
|
3453
|
+
- title: format_datetime
|
2370
3454
|
description: Converts a datetime or a datetime-string to the specified format.
|
2371
3455
|
arguments:
|
2372
3456
|
datetime: The datetime object or a string to convert. If datetime is a string, it must be formatted as ISO8601.
|
2373
|
-
format: The datetime format
|
3457
|
+
format: The datetime format.
|
3458
|
+
input_format: (optional) The datetime format in the case it is an string.
|
2374
3459
|
return_type: str
|
2375
3460
|
examples:
|
2376
3461
|
- "{{ format_datetime(config['start_time'], '%Y-%m-%d') }}"
|
2377
3462
|
- "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ') }}"
|
3463
|
+
- "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ', '%a, %d %b %Y %H:%M:%S %z') }}"
|
2378
3464
|
filters:
|
2379
|
-
- title:
|
3465
|
+
- title: hash
|
2380
3466
|
description: Convert the specified value to a hashed string.
|
2381
3467
|
arguments:
|
2382
3468
|
hash_type: Valid hash type for converts ('md5' as default value).
|
@@ -2386,15 +3472,29 @@ interpolation:
|
|
2386
3472
|
- "{{ 'Test client_secret' | hash() }} -> '3032d57a12f76b61a820e47b9a5a0cbb'"
|
2387
3473
|
- "{{ 'Test client_secret' | hash('md5') }} -> '3032d57a12f76b61a820e47b9a5a0cbb'"
|
2388
3474
|
- "{{ 'Test client_secret' | hash('md5', salt='salt') }} -> '5011a0168579c2d94cbbe1c6ad14327c'"
|
2389
|
-
- title:
|
3475
|
+
- title: base64encode
|
2390
3476
|
description: Convert the specified value to a string in the base64 format.
|
2391
3477
|
arguments: {}
|
2392
3478
|
return_type: str
|
2393
3479
|
examples:
|
2394
3480
|
- "{{ 'Test client_secret' | base64encode }} -> 'VGVzdCBjbGllbnRfc2VjcmV0'"
|
2395
|
-
- title:
|
3481
|
+
- title: base64decode
|
2396
3482
|
description: Decodes the specified base64 format value into a common string.
|
2397
3483
|
arguments: {}
|
2398
3484
|
return_type: str
|
2399
3485
|
examples:
|
2400
3486
|
- "{{ 'ZmFrZSByZWZyZXNoX3Rva2VuIHZhbHVl' | base64decode }} -> 'fake refresh_token value'"
|
3487
|
+
- title: string
|
3488
|
+
description: Converts the specified value to a string.
|
3489
|
+
arguments: {}
|
3490
|
+
return_type: str
|
3491
|
+
examples:
|
3492
|
+
- '{{ 1 | string }} -> "1"'
|
3493
|
+
- '{{ ["hello", "world" | string }} -> "["hello", "world"]"'
|
3494
|
+
- title: regex_search
|
3495
|
+
description: Match the input string against a regular expression and return the first match.
|
3496
|
+
arguments:
|
3497
|
+
regex: The regular expression to search for. It must include a capture group.
|
3498
|
+
return_type: str
|
3499
|
+
examples:
|
3500
|
+
- '{{ "goodbye, cruel world" | regex_search("goodbye,\s(.*)$") }} -> "cruel world"'
|