airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- airbyte_cdk/__init__.py +355 -6
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +29 -10
- airbyte_cdk/connector.py +24 -24
- airbyte_cdk/connector_builder/README.md +53 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
- airbyte_cdk/connector_builder/main.py +45 -13
- airbyte_cdk/connector_builder/message_grouper.py +189 -50
- airbyte_cdk/connector_builder/models.py +3 -2
- airbyte_cdk/destinations/__init__.py +4 -3
- airbyte_cdk/destinations/destination.py +54 -20
- airbyte_cdk/destinations/vector_db_based/README.md +37 -0
- airbyte_cdk/destinations/vector_db_based/config.py +40 -17
- airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
- airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
- airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
- airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
- airbyte_cdk/entrypoint.py +153 -44
- airbyte_cdk/exception_handler.py +21 -3
- airbyte_cdk/logger.py +30 -44
- airbyte_cdk/models/__init__.py +13 -2
- airbyte_cdk/models/airbyte_protocol.py +86 -1
- airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
- airbyte_cdk/models/file_transfer_record_message.py +13 -0
- airbyte_cdk/models/well_known_types.py +1 -1
- airbyte_cdk/sources/__init__.py +5 -1
- airbyte_cdk/sources/abstract_source.py +125 -79
- airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
- airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
- airbyte_cdk/sources/config.py +3 -2
- airbyte_cdk/sources/connector_state_manager.py +49 -83
- airbyte_cdk/sources/declarative/async_job/job.py +52 -0
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
- airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
- airbyte_cdk/sources/declarative/async_job/status.py +24 -0
- airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
- airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
- airbyte_cdk/sources/declarative/auth/token.py +28 -10
- airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
- airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
- airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
- airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1213 -88
- airbyte_cdk/sources/declarative/declarative_source.py +5 -2
- airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
- airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
- airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
- airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
- airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
- airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
- airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
- airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
- airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
- airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +346 -0
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
- airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +173 -74
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
- airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
- airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
- airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
- airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
- airbyte_cdk/sources/declarative/models/__init__.py +1 -1
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1329 -595
- airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1763 -226
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
- airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
- airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
- airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
- airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
- airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
- airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
- airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
- airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
- airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
- airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
- airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
- airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
- airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
- airbyte_cdk/sources/declarative/spec/spec.py +12 -5
- airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
- airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
- airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
- airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
- airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
- airbyte_cdk/sources/declarative/types.py +19 -110
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
- airbyte_cdk/sources/embedded/base_integration.py +16 -5
- airbyte_cdk/sources/embedded/catalog.py +16 -4
- airbyte_cdk/sources/embedded/runner.py +19 -3
- airbyte_cdk/sources/embedded/tools.py +5 -2
- airbyte_cdk/sources/file_based/README.md +152 -0
- airbyte_cdk/sources/file_based/__init__.py +24 -0
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
- airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
- airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
- airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
- airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
- airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
- airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
- airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
- airbyte_cdk/sources/file_based/exceptions.py +18 -15
- airbyte_cdk/sources/file_based/file_based_source.py +140 -33
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
- airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
- airbyte_cdk/sources/file_based/remote_file.py +1 -1
- airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
- airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
- airbyte_cdk/sources/http_logger.py +8 -3
- airbyte_cdk/sources/message/__init__.py +7 -1
- airbyte_cdk/sources/message/repository.py +18 -4
- airbyte_cdk/sources/source.py +42 -38
- airbyte_cdk/sources/streams/__init__.py +2 -2
- airbyte_cdk/sources/streams/availability_strategy.py +54 -3
- airbyte_cdk/sources/streams/call_rate.py +64 -21
- airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
- airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
- airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
- airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
- airbyte_cdk/sources/streams/concurrent/README.md +7 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
- airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
- airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
- airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
- airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
- airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
- airbyte_cdk/sources/streams/core.py +412 -87
- airbyte_cdk/sources/streams/http/__init__.py +2 -1
- airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
- airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
- airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
- airbyte_cdk/sources/streams/http/exceptions.py +27 -7
- airbyte_cdk/sources/streams/http/http.py +369 -246
- airbyte_cdk/sources/streams/http/http_client.py +531 -0
- airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
- airbyte_cdk/sources/types.py +154 -0
- airbyte_cdk/sources/utils/record_helper.py +36 -21
- airbyte_cdk/sources/utils/schema_helpers.py +13 -6
- airbyte_cdk/sources/utils/slice_logger.py +4 -1
- airbyte_cdk/sources/utils/transform.py +54 -20
- airbyte_cdk/sql/_util/hashing.py +34 -0
- airbyte_cdk/sql/_util/name_normalizers.py +92 -0
- airbyte_cdk/sql/constants.py +32 -0
- airbyte_cdk/sql/exceptions.py +235 -0
- airbyte_cdk/sql/secrets.py +123 -0
- airbyte_cdk/sql/shared/__init__.py +15 -0
- airbyte_cdk/sql/shared/catalog_providers.py +145 -0
- airbyte_cdk/sql/shared/sql_processor.py +786 -0
- airbyte_cdk/sql/types.py +160 -0
- airbyte_cdk/test/catalog_builder.py +70 -18
- airbyte_cdk/test/entrypoint_wrapper.py +117 -42
- airbyte_cdk/test/mock_http/__init__.py +1 -1
- airbyte_cdk/test/mock_http/matcher.py +6 -0
- airbyte_cdk/test/mock_http/mocker.py +57 -10
- airbyte_cdk/test/mock_http/request.py +19 -3
- airbyte_cdk/test/mock_http/response.py +3 -1
- airbyte_cdk/test/mock_http/response_builder.py +32 -16
- airbyte_cdk/test/state_builder.py +18 -10
- airbyte_cdk/test/utils/__init__.py +1 -0
- airbyte_cdk/test/utils/data.py +24 -0
- airbyte_cdk/test/utils/http_mocking.py +16 -0
- airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
- airbyte_cdk/test/utils/reading.py +26 -0
- airbyte_cdk/utils/__init__.py +2 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
- airbyte_cdk/utils/analytics_message.py +10 -2
- airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
- airbyte_cdk/utils/event_timing.py +10 -10
- airbyte_cdk/utils/mapping_helpers.py +3 -1
- airbyte_cdk/utils/message_utils.py +20 -11
- airbyte_cdk/utils/print_buffer.py +75 -0
- airbyte_cdk/utils/schema_inferrer.py +198 -28
- airbyte_cdk/utils/slice_hasher.py +30 -0
- airbyte_cdk/utils/spec_schema_transformations.py +6 -3
- airbyte_cdk/utils/stream_status_utils.py +8 -1
- airbyte_cdk/utils/traced_exception.py +61 -21
- airbyte_cdk-6.17.1.dev0.dist-info/METADATA +109 -0
- airbyte_cdk-6.17.1.dev0.dist-info/RECORD +350 -0
- {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/WHEEL +1 -2
- airbyte_cdk-6.17.1.dev0.dist-info/entry_points.txt +3 -0
- airbyte_cdk/sources/declarative/create_partial.py +0 -92
- airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
- airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
- airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
- airbyte_cdk/sources/deprecated/base_source.py +0 -94
- airbyte_cdk/sources/deprecated/client.py +0 -99
- airbyte_cdk/sources/singer/__init__.py +0 -8
- airbyte_cdk/sources/singer/singer_helpers.py +0 -304
- airbyte_cdk/sources/singer/source.py +0 -186
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
- airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
- airbyte_cdk/sources/streams/http/auth/core.py +0 -29
- airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
- airbyte_cdk/sources/streams/http/auth/token.py +0 -47
- airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
- airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
- airbyte_cdk/sources/utils/schema_models.py +0 -84
- airbyte_cdk-0.72.0.dist-info/METADATA +0 -243
- airbyte_cdk-0.72.0.dist-info/RECORD +0 -466
- airbyte_cdk-0.72.0.dist-info/top_level.txt +0 -3
- source_declarative_manifest/main.py +0 -29
- unit_tests/connector_builder/__init__.py +0 -3
- unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
- unit_tests/connector_builder/test_message_grouper.py +0 -713
- unit_tests/connector_builder/utils.py +0 -27
- unit_tests/destinations/test_destination.py +0 -243
- unit_tests/singer/test_singer_helpers.py +0 -56
- unit_tests/singer/test_singer_source.py +0 -112
- unit_tests/sources/__init__.py +0 -0
- unit_tests/sources/concurrent_source/__init__.py +0 -3
- unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
- unit_tests/sources/declarative/__init__.py +0 -3
- unit_tests/sources/declarative/auth/__init__.py +0 -3
- unit_tests/sources/declarative/auth/test_oauth.py +0 -331
- unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
- unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
- unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
- unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
- unit_tests/sources/declarative/checks/__init__.py +0 -3
- unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
- unit_tests/sources/declarative/decoders/__init__.py +0 -0
- unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
- unit_tests/sources/declarative/external_component.py +0 -13
- unit_tests/sources/declarative/extractors/__init__.py +0 -3
- unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
- unit_tests/sources/declarative/incremental/__init__.py +0 -0
- unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
- unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
- unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
- unit_tests/sources/declarative/interpolation/__init__.py +0 -3
- unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
- unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
- unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
- unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
- unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
- unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
- unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
- unit_tests/sources/declarative/parsers/__init__.py +0 -3
- unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
- unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
- unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1841
- unit_tests/sources/declarative/parsers/testing_components.py +0 -36
- unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
- unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
- unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
- unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
- unit_tests/sources/declarative/requesters/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
- unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
- unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
- unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
- unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
- unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
- unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
- unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
- unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
- unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
- unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
- unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
- unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
- unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
- unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
- unit_tests/sources/declarative/retrievers/__init__.py +0 -3
- unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
- unit_tests/sources/declarative/schema/__init__.py +0 -6
- unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
- unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
- unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
- unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
- unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
- unit_tests/sources/declarative/states/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
- unit_tests/sources/declarative/test_create_partial.py +0 -83
- unit_tests/sources/declarative/test_declarative_stream.py +0 -103
- unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
- unit_tests/sources/declarative/test_types.py +0 -39
- unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
- unit_tests/sources/file_based/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
- unit_tests/sources/file_based/config/__init__.py +0 -0
- unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
- unit_tests/sources/file_based/config/test_csv_format.py +0 -34
- unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
- unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
- unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
- unit_tests/sources/file_based/file_types/__init__.py +0 -0
- unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
- unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
- unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
- unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
- unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
- unit_tests/sources/file_based/helpers.py +0 -70
- unit_tests/sources/file_based/in_memory_files_source.py +0 -211
- unit_tests/sources/file_based/scenarios/__init__.py +0 -0
- unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
- unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
- unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
- unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
- unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
- unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
- unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
- unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
- unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
- unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
- unit_tests/sources/file_based/stream/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
- unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
- unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
- unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
- unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
- unit_tests/sources/file_based/test_scenarios.py +0 -253
- unit_tests/sources/file_based/test_schema_helpers.py +0 -346
- unit_tests/sources/fixtures/__init__.py +0 -3
- unit_tests/sources/fixtures/source_test_fixture.py +0 -153
- unit_tests/sources/message/__init__.py +0 -0
- unit_tests/sources/message/test_repository.py +0 -153
- unit_tests/sources/streams/__init__.py +0 -0
- unit_tests/sources/streams/concurrent/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
- unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
- unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
- unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
- unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
- unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
- unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
- unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
- unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
- unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
- unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
- unit_tests/sources/streams/http/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/test_auth.py +0 -173
- unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
- unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
- unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
- unit_tests/sources/streams/http/test_http.py +0 -635
- unit_tests/sources/streams/test_availability_strategy.py +0 -70
- unit_tests/sources/streams/test_call_rate.py +0 -300
- unit_tests/sources/streams/test_stream_read.py +0 -405
- unit_tests/sources/streams/test_streams_core.py +0 -184
- unit_tests/sources/test_abstract_source.py +0 -1442
- unit_tests/sources/test_concurrent_source.py +0 -112
- unit_tests/sources/test_config.py +0 -92
- unit_tests/sources/test_connector_state_manager.py +0 -482
- unit_tests/sources/test_http_logger.py +0 -252
- unit_tests/sources/test_integration_source.py +0 -86
- unit_tests/sources/test_source.py +0 -684
- unit_tests/sources/test_source_read.py +0 -460
- unit_tests/test/__init__.py +0 -0
- unit_tests/test/mock_http/__init__.py +0 -0
- unit_tests/test/mock_http/test_matcher.py +0 -53
- unit_tests/test/mock_http/test_mocker.py +0 -214
- unit_tests/test/mock_http/test_request.py +0 -117
- unit_tests/test/mock_http/test_response_builder.py +0 -177
- unit_tests/test/test_entrypoint_wrapper.py +0 -240
- unit_tests/utils/__init__.py +0 -0
- unit_tests/utils/test_datetime_format_inferrer.py +0 -60
- unit_tests/utils/test_mapping_helpers.py +0 -54
- unit_tests/utils/test_message_utils.py +0 -91
- unit_tests/utils/test_rate_limiting.py +0 -26
- unit_tests/utils/test_schema_inferrer.py +0 -202
- unit_tests/utils/test_secret_utils.py +0 -135
- unit_tests/utils/test_stream_status_utils.py +0 -61
- unit_tests/utils/test_traced_exception.py +0 -107
- /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
- {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
- {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
- {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
- {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/LICENSE.txt +0 -0
@@ -1,1841 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
# mypy: ignore-errors
|
6
|
-
|
7
|
-
import datetime
|
8
|
-
from typing import Any, Mapping
|
9
|
-
|
10
|
-
import pytest
|
11
|
-
from airbyte_cdk.models import Level
|
12
|
-
from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator
|
13
|
-
from airbyte_cdk.sources.declarative.auth.token import (
|
14
|
-
ApiKeyAuthenticator,
|
15
|
-
BasicHttpAuthenticator,
|
16
|
-
BearerAuthenticator,
|
17
|
-
LegacySessionTokenAuthenticator,
|
18
|
-
)
|
19
|
-
from airbyte_cdk.sources.declarative.auth.token_provider import SessionTokenProvider
|
20
|
-
from airbyte_cdk.sources.declarative.checks import CheckStream
|
21
|
-
from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime
|
22
|
-
from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
|
23
|
-
from airbyte_cdk.sources.declarative.decoders import JsonDecoder
|
24
|
-
from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordFilter, RecordSelector
|
25
|
-
from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor, PerPartitionCursor
|
26
|
-
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
27
|
-
from airbyte_cdk.sources.declarative.models import CheckStream as CheckStreamModel
|
28
|
-
from airbyte_cdk.sources.declarative.models import CompositeErrorHandler as CompositeErrorHandlerModel
|
29
|
-
from airbyte_cdk.sources.declarative.models import CustomErrorHandler as CustomErrorHandlerModel
|
30
|
-
from airbyte_cdk.sources.declarative.models import CustomPartitionRouter as CustomPartitionRouterModel
|
31
|
-
from airbyte_cdk.sources.declarative.models import CustomSchemaLoader as CustomSchemaLoaderModel
|
32
|
-
from airbyte_cdk.sources.declarative.models import DatetimeBasedCursor as DatetimeBasedCursorModel
|
33
|
-
from airbyte_cdk.sources.declarative.models import DeclarativeStream as DeclarativeStreamModel
|
34
|
-
from airbyte_cdk.sources.declarative.models import DefaultPaginator as DefaultPaginatorModel
|
35
|
-
from airbyte_cdk.sources.declarative.models import HttpRequester as HttpRequesterModel
|
36
|
-
from airbyte_cdk.sources.declarative.models import ListPartitionRouter as ListPartitionRouterModel
|
37
|
-
from airbyte_cdk.sources.declarative.models import OAuthAuthenticator as OAuthAuthenticatorModel
|
38
|
-
from airbyte_cdk.sources.declarative.models import RecordSelector as RecordSelectorModel
|
39
|
-
from airbyte_cdk.sources.declarative.models import SimpleRetriever as SimpleRetrieverModel
|
40
|
-
from airbyte_cdk.sources.declarative.models import Spec as SpecModel
|
41
|
-
from airbyte_cdk.sources.declarative.models import SubstreamPartitionRouter as SubstreamPartitionRouterModel
|
42
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import OffsetIncrement as OffsetIncrementModel
|
43
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import PageIncrement as PageIncrementModel
|
44
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import SelectiveAuthenticator
|
45
|
-
from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer
|
46
|
-
from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver
|
47
|
-
from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory
|
48
|
-
from airbyte_cdk.sources.declarative.partition_routers import ListPartitionRouter, SinglePartitionRouter, SubstreamPartitionRouter
|
49
|
-
from airbyte_cdk.sources.declarative.requesters import HttpRequester
|
50
|
-
from airbyte_cdk.sources.declarative.requesters.error_handlers import CompositeErrorHandler, DefaultErrorHandler, HttpResponseFilter
|
51
|
-
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import (
|
52
|
-
ConstantBackoffStrategy,
|
53
|
-
ExponentialBackoffStrategy,
|
54
|
-
WaitTimeFromHeaderBackoffStrategy,
|
55
|
-
WaitUntilTimeFromHeaderBackoffStrategy,
|
56
|
-
)
|
57
|
-
from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction
|
58
|
-
from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator
|
59
|
-
from airbyte_cdk.sources.declarative.requesters.paginators.strategies import (
|
60
|
-
CursorPaginationStrategy,
|
61
|
-
OffsetIncrement,
|
62
|
-
PageIncrement,
|
63
|
-
StopConditionPaginationStrategyDecorator,
|
64
|
-
)
|
65
|
-
from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType
|
66
|
-
from airbyte_cdk.sources.declarative.requesters.request_options import InterpolatedRequestOptionsProvider
|
67
|
-
from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
|
68
|
-
from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
|
69
|
-
from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever, SimpleRetrieverTestReadDecorator
|
70
|
-
from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader
|
71
|
-
from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
|
72
|
-
from airbyte_cdk.sources.declarative.spec import Spec
|
73
|
-
from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStreamSlicer
|
74
|
-
from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields
|
75
|
-
from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
|
76
|
-
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
|
77
|
-
from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator
|
78
|
-
from unit_tests.sources.declarative.parsers.testing_components import TestingCustomSubstreamPartitionRouter, TestingSomeComponent
|
79
|
-
|
80
|
-
factory = ModelToComponentFactory()
|
81
|
-
|
82
|
-
resolver = ManifestReferenceResolver()
|
83
|
-
|
84
|
-
transformer = ManifestComponentTransformer()
|
85
|
-
|
86
|
-
input_config = {"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}
|
87
|
-
|
88
|
-
|
89
|
-
def test_create_check_stream():
|
90
|
-
manifest = {"check": {"type": "CheckStream", "stream_names": ["list_stream"]}}
|
91
|
-
|
92
|
-
check = factory.create_component(CheckStreamModel, manifest["check"], {})
|
93
|
-
|
94
|
-
assert isinstance(check, CheckStream)
|
95
|
-
assert check.stream_names == ["list_stream"]
|
96
|
-
|
97
|
-
|
98
|
-
def test_create_component_type_mismatch():
|
99
|
-
manifest = {"check": {"type": "MismatchType", "stream_names": ["list_stream"]}}
|
100
|
-
|
101
|
-
with pytest.raises(ValueError):
|
102
|
-
factory.create_component(CheckStreamModel, manifest["check"], {})
|
103
|
-
|
104
|
-
|
105
|
-
def test_full_config_stream():
|
106
|
-
content = """
|
107
|
-
decoder:
|
108
|
-
type: JsonDecoder
|
109
|
-
extractor:
|
110
|
-
type: DpathExtractor
|
111
|
-
decoder: "#/decoder"
|
112
|
-
selector:
|
113
|
-
type: RecordSelector
|
114
|
-
record_filter:
|
115
|
-
type: RecordFilter
|
116
|
-
condition: "{{ record['id'] > stream_state['id'] }}"
|
117
|
-
metadata_paginator:
|
118
|
-
type: DefaultPaginator
|
119
|
-
page_size_option:
|
120
|
-
type: RequestOption
|
121
|
-
inject_into: request_parameter
|
122
|
-
field_name: page_size
|
123
|
-
page_token_option:
|
124
|
-
type: RequestPath
|
125
|
-
pagination_strategy:
|
126
|
-
type: "CursorPagination"
|
127
|
-
cursor_value: "{{ response._metadata.next }}"
|
128
|
-
page_size: 10
|
129
|
-
requester:
|
130
|
-
type: HttpRequester
|
131
|
-
url_base: "https://api.sendgrid.com/v3/"
|
132
|
-
http_method: "GET"
|
133
|
-
authenticator:
|
134
|
-
type: BearerAuthenticator
|
135
|
-
api_token: "{{ config['apikey'] }}"
|
136
|
-
request_parameters:
|
137
|
-
unit: "day"
|
138
|
-
retriever:
|
139
|
-
paginator:
|
140
|
-
type: NoPagination
|
141
|
-
partial_stream:
|
142
|
-
type: DeclarativeStream
|
143
|
-
schema_loader:
|
144
|
-
type: JsonFileSchemaLoader
|
145
|
-
file_path: "./source_sendgrid/schemas/{{ parameters.name }}.json"
|
146
|
-
list_stream:
|
147
|
-
$ref: "#/partial_stream"
|
148
|
-
$parameters:
|
149
|
-
name: "lists"
|
150
|
-
extractor:
|
151
|
-
$ref: "#/extractor"
|
152
|
-
field_path: ["{{ parameters['name'] }}"]
|
153
|
-
name: "lists"
|
154
|
-
primary_key: "id"
|
155
|
-
retriever:
|
156
|
-
$ref: "#/retriever"
|
157
|
-
requester:
|
158
|
-
$ref: "#/requester"
|
159
|
-
path: "{{ next_page_token['next_page_url'] }}"
|
160
|
-
paginator:
|
161
|
-
$ref: "#/metadata_paginator"
|
162
|
-
record_selector:
|
163
|
-
$ref: "#/selector"
|
164
|
-
transformations:
|
165
|
-
- type: AddFields
|
166
|
-
fields:
|
167
|
-
- path: ["extra"]
|
168
|
-
value: "{{ response.to_add }}"
|
169
|
-
incremental_sync:
|
170
|
-
type: DatetimeBasedCursor
|
171
|
-
start_datetime: "{{ config['start_time'] }}"
|
172
|
-
end_datetime: "{{ config['end_time'] }}"
|
173
|
-
step: "P10D"
|
174
|
-
cursor_field: "created"
|
175
|
-
cursor_granularity: "PT0.000001S"
|
176
|
-
$parameters:
|
177
|
-
datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
|
178
|
-
check:
|
179
|
-
type: CheckStream
|
180
|
-
stream_names: ["list_stream"]
|
181
|
-
spec:
|
182
|
-
type: Spec
|
183
|
-
documentation_url: https://airbyte.com/#yaml-from-manifest
|
184
|
-
connection_specification:
|
185
|
-
title: Test Spec
|
186
|
-
type: object
|
187
|
-
required:
|
188
|
-
- api_key
|
189
|
-
additionalProperties: false
|
190
|
-
properties:
|
191
|
-
api_key:
|
192
|
-
type: string
|
193
|
-
airbyte_secret: true
|
194
|
-
title: API Key
|
195
|
-
description: Test API Key
|
196
|
-
order: 0
|
197
|
-
advanced_auth:
|
198
|
-
auth_flow_type: "oauth2.0"
|
199
|
-
"""
|
200
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
201
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
202
|
-
resolved_manifest["type"] = "DeclarativeSource"
|
203
|
-
manifest = transformer.propagate_types_and_parameters("", resolved_manifest, {})
|
204
|
-
|
205
|
-
stream_manifest = manifest["list_stream"]
|
206
|
-
assert stream_manifest["type"] == "DeclarativeStream"
|
207
|
-
stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
|
208
|
-
|
209
|
-
assert isinstance(stream, DeclarativeStream)
|
210
|
-
assert stream.primary_key == "id"
|
211
|
-
assert stream.name == "lists"
|
212
|
-
assert stream._stream_cursor_field.string == "created"
|
213
|
-
|
214
|
-
assert isinstance(stream.schema_loader, JsonFileSchemaLoader)
|
215
|
-
assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.json"
|
216
|
-
|
217
|
-
assert len(stream.retriever.record_selector.transformations) == 1
|
218
|
-
add_fields = stream.retriever.record_selector.transformations[0]
|
219
|
-
assert isinstance(add_fields, AddFields)
|
220
|
-
assert add_fields.fields[0].path == ["extra"]
|
221
|
-
assert add_fields.fields[0].value.string == "{{ response.to_add }}"
|
222
|
-
|
223
|
-
assert isinstance(stream.retriever, SimpleRetriever)
|
224
|
-
assert stream.retriever.primary_key == stream.primary_key
|
225
|
-
assert stream.retriever.name == stream.name
|
226
|
-
|
227
|
-
assert isinstance(stream.retriever.record_selector, RecordSelector)
|
228
|
-
|
229
|
-
assert isinstance(stream.retriever.record_selector.extractor, DpathExtractor)
|
230
|
-
assert isinstance(stream.retriever.record_selector.extractor.decoder, JsonDecoder)
|
231
|
-
assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_path] == ["lists"]
|
232
|
-
|
233
|
-
assert isinstance(stream.retriever.record_selector.record_filter, RecordFilter)
|
234
|
-
assert stream.retriever.record_selector.record_filter._filter_interpolator.condition == "{{ record['id'] > stream_state['id'] }}"
|
235
|
-
|
236
|
-
assert isinstance(stream.retriever.paginator, DefaultPaginator)
|
237
|
-
assert isinstance(stream.retriever.paginator.decoder, JsonDecoder)
|
238
|
-
assert stream.retriever.paginator.page_size_option.field_name.eval(input_config) == "page_size"
|
239
|
-
assert stream.retriever.paginator.page_size_option.inject_into == RequestOptionType.request_parameter
|
240
|
-
assert isinstance(stream.retriever.paginator.page_token_option, RequestPath)
|
241
|
-
assert stream.retriever.paginator.url_base.string == "https://api.sendgrid.com/v3/"
|
242
|
-
assert stream.retriever.paginator.url_base.default == "https://api.sendgrid.com/v3/"
|
243
|
-
|
244
|
-
assert isinstance(stream.retriever.paginator.pagination_strategy, CursorPaginationStrategy)
|
245
|
-
assert isinstance(stream.retriever.paginator.pagination_strategy.decoder, JsonDecoder)
|
246
|
-
assert stream.retriever.paginator.pagination_strategy.cursor_value.string == "{{ response._metadata.next }}"
|
247
|
-
assert stream.retriever.paginator.pagination_strategy.cursor_value.default == "{{ response._metadata.next }}"
|
248
|
-
assert stream.retriever.paginator.pagination_strategy.page_size == 10
|
249
|
-
|
250
|
-
assert isinstance(stream.retriever.requester, HttpRequester)
|
251
|
-
assert stream.retriever.requester.http_method == HttpMethod.GET
|
252
|
-
assert stream.retriever.requester.name == stream.name
|
253
|
-
assert stream.retriever.requester._path.string == "{{ next_page_token['next_page_url'] }}"
|
254
|
-
assert stream.retriever.requester._path.default == "{{ next_page_token['next_page_url'] }}"
|
255
|
-
|
256
|
-
assert isinstance(stream.retriever.requester.authenticator, BearerAuthenticator)
|
257
|
-
assert stream.retriever.requester.authenticator.token_provider.get_token() == "verysecrettoken"
|
258
|
-
|
259
|
-
assert isinstance(stream.retriever.requester.request_options_provider, InterpolatedRequestOptionsProvider)
|
260
|
-
assert stream.retriever.requester.request_options_provider.request_parameters.get("unit") == "day"
|
261
|
-
|
262
|
-
checker = factory.create_component(model_type=CheckStreamModel, component_definition=manifest["check"], config=input_config)
|
263
|
-
|
264
|
-
assert isinstance(checker, CheckStream)
|
265
|
-
streams_to_check = checker.stream_names
|
266
|
-
assert len(streams_to_check) == 1
|
267
|
-
assert list(streams_to_check)[0] == "list_stream"
|
268
|
-
|
269
|
-
spec = factory.create_component(model_type=SpecModel, component_definition=manifest["spec"], config=input_config)
|
270
|
-
|
271
|
-
assert isinstance(spec, Spec)
|
272
|
-
documentation_url = spec.documentation_url
|
273
|
-
connection_specification = spec.connection_specification
|
274
|
-
assert documentation_url == "https://airbyte.com/#yaml-from-manifest"
|
275
|
-
assert connection_specification["title"] == "Test Spec"
|
276
|
-
assert connection_specification["required"] == ["api_key"]
|
277
|
-
assert connection_specification["properties"]["api_key"] == {
|
278
|
-
"type": "string",
|
279
|
-
"airbyte_secret": True,
|
280
|
-
"title": "API Key",
|
281
|
-
"description": "Test API Key",
|
282
|
-
"order": 0,
|
283
|
-
}
|
284
|
-
advanced_auth = spec.advanced_auth
|
285
|
-
assert advanced_auth.auth_flow_type.value == "oauth2.0"
|
286
|
-
|
287
|
-
|
288
|
-
def test_interpolate_config():
|
289
|
-
content = """
|
290
|
-
authenticator:
|
291
|
-
type: OAuthAuthenticator
|
292
|
-
client_id: "some_client_id"
|
293
|
-
client_secret: "some_client_secret"
|
294
|
-
token_refresh_endpoint: "https://api.sendgrid.com/v3/auth"
|
295
|
-
refresh_token: "{{ config['apikey'] }}"
|
296
|
-
refresh_request_body:
|
297
|
-
body_field: "yoyoyo"
|
298
|
-
interpolated_body_field: "{{ config['apikey'] }}"
|
299
|
-
"""
|
300
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
301
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
302
|
-
authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
|
303
|
-
|
304
|
-
authenticator = factory.create_component(
|
305
|
-
model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=input_config
|
306
|
-
)
|
307
|
-
|
308
|
-
assert isinstance(authenticator, DeclarativeOauth2Authenticator)
|
309
|
-
assert authenticator._client_id.eval(input_config) == "some_client_id"
|
310
|
-
assert authenticator._client_secret.string == "some_client_secret"
|
311
|
-
assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth"
|
312
|
-
assert authenticator._refresh_token.eval(input_config) == "verysecrettoken"
|
313
|
-
assert authenticator._refresh_request_body.mapping == {"body_field": "yoyoyo", "interpolated_body_field": "{{ config['apikey'] }}"}
|
314
|
-
assert authenticator.get_refresh_request_body() == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"}
|
315
|
-
|
316
|
-
|
317
|
-
def test_interpolate_config_with_token_expiry_date_format():
|
318
|
-
content = """
|
319
|
-
authenticator:
|
320
|
-
type: OAuthAuthenticator
|
321
|
-
client_id: "some_client_id"
|
322
|
-
client_secret: "some_client_secret"
|
323
|
-
token_refresh_endpoint: "https://api.sendgrid.com/v3/auth"
|
324
|
-
refresh_token: "{{ config['apikey'] }}"
|
325
|
-
token_expiry_date_format: "%Y-%m-%d %H:%M:%S.%f+00:00"
|
326
|
-
"""
|
327
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
328
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
329
|
-
authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
|
330
|
-
|
331
|
-
authenticator = factory.create_component(
|
332
|
-
model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=input_config
|
333
|
-
)
|
334
|
-
|
335
|
-
assert isinstance(authenticator, DeclarativeOauth2Authenticator)
|
336
|
-
assert authenticator.token_expiry_date_format == "%Y-%m-%d %H:%M:%S.%f+00:00"
|
337
|
-
assert authenticator.token_expiry_is_time_of_expiration
|
338
|
-
assert authenticator._client_id.eval(input_config) == "some_client_id"
|
339
|
-
assert authenticator._client_secret.string == "some_client_secret"
|
340
|
-
assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth"
|
341
|
-
|
342
|
-
|
343
|
-
def test_single_use_oauth_branch():
|
344
|
-
single_use_input_config = {
|
345
|
-
"apikey": "verysecrettoken",
|
346
|
-
"repos": ["airbyte", "airbyte-cloud"],
|
347
|
-
"credentials": {"access_token": "access_token", "token_expiry_date": "1970-01-01"},
|
348
|
-
}
|
349
|
-
|
350
|
-
content = """
|
351
|
-
authenticator:
|
352
|
-
type: OAuthAuthenticator
|
353
|
-
client_id: "some_client_id"
|
354
|
-
client_secret: "some_client_secret"
|
355
|
-
token_refresh_endpoint: "https://api.sendgrid.com/v3/auth"
|
356
|
-
refresh_token: "{{ config['apikey'] }}"
|
357
|
-
refresh_request_body:
|
358
|
-
body_field: "yoyoyo"
|
359
|
-
interpolated_body_field: "{{ config['apikey'] }}"
|
360
|
-
refresh_token_updater:
|
361
|
-
refresh_token_name: "the_refresh_token"
|
362
|
-
refresh_token_config_path:
|
363
|
-
- apikey
|
364
|
-
"""
|
365
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
366
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
367
|
-
authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
|
368
|
-
|
369
|
-
authenticator: SingleUseRefreshTokenOauth2Authenticator = factory.create_component(
|
370
|
-
model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=single_use_input_config
|
371
|
-
)
|
372
|
-
|
373
|
-
assert isinstance(authenticator, SingleUseRefreshTokenOauth2Authenticator)
|
374
|
-
assert authenticator._client_id == "some_client_id"
|
375
|
-
assert authenticator._client_secret == "some_client_secret"
|
376
|
-
assert authenticator._token_refresh_endpoint == "https://api.sendgrid.com/v3/auth"
|
377
|
-
assert authenticator._refresh_token == "verysecrettoken"
|
378
|
-
assert authenticator._refresh_request_body == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"}
|
379
|
-
assert authenticator._refresh_token_name == "the_refresh_token"
|
380
|
-
assert authenticator._refresh_token_config_path == ["apikey"]
|
381
|
-
# default values
|
382
|
-
assert authenticator._access_token_config_path == ["credentials", "access_token"]
|
383
|
-
assert authenticator._token_expiry_date_config_path == ["credentials", "token_expiry_date"]
|
384
|
-
|
385
|
-
|
386
|
-
def test_list_based_stream_slicer_with_values_refd():
|
387
|
-
content = """
|
388
|
-
repositories: ["airbyte", "airbyte-cloud"]
|
389
|
-
partition_router:
|
390
|
-
type: ListPartitionRouter
|
391
|
-
values: "#/repositories"
|
392
|
-
cursor_field: repository
|
393
|
-
"""
|
394
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
395
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
396
|
-
partition_router_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["partition_router"], {})
|
397
|
-
|
398
|
-
partition_router = factory.create_component(
|
399
|
-
model_type=ListPartitionRouterModel, component_definition=partition_router_manifest, config=input_config
|
400
|
-
)
|
401
|
-
|
402
|
-
assert isinstance(partition_router, ListPartitionRouter)
|
403
|
-
assert partition_router.values == ["airbyte", "airbyte-cloud"]
|
404
|
-
|
405
|
-
|
406
|
-
def test_list_based_stream_slicer_with_values_defined_in_config():
|
407
|
-
content = """
|
408
|
-
partition_router:
|
409
|
-
type: ListPartitionRouter
|
410
|
-
values: "{{config['repos']}}"
|
411
|
-
cursor_field: repository
|
412
|
-
request_option:
|
413
|
-
type: RequestOption
|
414
|
-
inject_into: header
|
415
|
-
field_name: repository
|
416
|
-
"""
|
417
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
418
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
419
|
-
partition_router_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["partition_router"], {})
|
420
|
-
|
421
|
-
partition_router = factory.create_component(
|
422
|
-
model_type=ListPartitionRouterModel, component_definition=partition_router_manifest, config=input_config
|
423
|
-
)
|
424
|
-
|
425
|
-
assert isinstance(partition_router, ListPartitionRouter)
|
426
|
-
assert partition_router.values == ["airbyte", "airbyte-cloud"]
|
427
|
-
assert partition_router.request_option.inject_into == RequestOptionType.header
|
428
|
-
assert partition_router.request_option.field_name.eval(config=input_config) == "repository"
|
429
|
-
|
430
|
-
|
431
|
-
def test_create_substream_partition_router():
|
432
|
-
content = """
|
433
|
-
schema_loader:
|
434
|
-
file_path: "./source_sendgrid/schemas/{{ parameters['name'] }}.yaml"
|
435
|
-
name: "{{ parameters['stream_name'] }}"
|
436
|
-
retriever:
|
437
|
-
requester:
|
438
|
-
type: "HttpRequester"
|
439
|
-
path: "kek"
|
440
|
-
record_selector:
|
441
|
-
extractor:
|
442
|
-
field_path: []
|
443
|
-
stream_A:
|
444
|
-
type: DeclarativeStream
|
445
|
-
name: "A"
|
446
|
-
primary_key: "id"
|
447
|
-
$parameters:
|
448
|
-
retriever: "#/retriever"
|
449
|
-
url_base: "https://airbyte.io"
|
450
|
-
schema_loader: "#/schema_loader"
|
451
|
-
stream_B:
|
452
|
-
type: DeclarativeStream
|
453
|
-
name: "B"
|
454
|
-
primary_key: "id"
|
455
|
-
$parameters:
|
456
|
-
retriever: "#/retriever"
|
457
|
-
url_base: "https://airbyte.io"
|
458
|
-
schema_loader: "#/schema_loader"
|
459
|
-
partition_router:
|
460
|
-
type: SubstreamPartitionRouter
|
461
|
-
parent_stream_configs:
|
462
|
-
- stream: "#/stream_A"
|
463
|
-
parent_key: id
|
464
|
-
partition_field: repository_id
|
465
|
-
request_option:
|
466
|
-
type: RequestOption
|
467
|
-
inject_into: request_parameter
|
468
|
-
field_name: repository_id
|
469
|
-
- stream: "#/stream_B"
|
470
|
-
parent_key: someid
|
471
|
-
partition_field: word_id
|
472
|
-
"""
|
473
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
474
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
475
|
-
partition_router_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["partition_router"], {})
|
476
|
-
|
477
|
-
partition_router = factory.create_component(
|
478
|
-
model_type=SubstreamPartitionRouterModel, component_definition=partition_router_manifest, config=input_config
|
479
|
-
)
|
480
|
-
|
481
|
-
assert isinstance(partition_router, SubstreamPartitionRouter)
|
482
|
-
parent_stream_configs = partition_router.parent_stream_configs
|
483
|
-
assert len(parent_stream_configs) == 2
|
484
|
-
assert isinstance(parent_stream_configs[0].stream, DeclarativeStream)
|
485
|
-
assert isinstance(parent_stream_configs[1].stream, DeclarativeStream)
|
486
|
-
|
487
|
-
assert partition_router.parent_stream_configs[0].parent_key.eval({}) == "id"
|
488
|
-
assert partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id"
|
489
|
-
assert partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter
|
490
|
-
assert partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id"
|
491
|
-
|
492
|
-
assert partition_router.parent_stream_configs[1].parent_key.eval({}) == "someid"
|
493
|
-
assert partition_router.parent_stream_configs[1].partition_field.eval({}) == "word_id"
|
494
|
-
assert partition_router.parent_stream_configs[1].request_option is None
|
495
|
-
|
496
|
-
|
497
|
-
def test_datetime_based_cursor():
|
498
|
-
content = """
|
499
|
-
incremental:
|
500
|
-
type: DatetimeBasedCursor
|
501
|
-
$parameters:
|
502
|
-
datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
|
503
|
-
start_datetime:
|
504
|
-
type: MinMaxDatetime
|
505
|
-
datetime: "{{ config['start_time'] }}"
|
506
|
-
min_datetime: "{{ config['start_time'] + day_delta(2) }}"
|
507
|
-
end_datetime: "{{ config['end_time'] }}"
|
508
|
-
step: "P10D"
|
509
|
-
cursor_field: "created"
|
510
|
-
cursor_granularity: "PT0.000001S"
|
511
|
-
lookback_window: "P5D"
|
512
|
-
start_time_option:
|
513
|
-
type: RequestOption
|
514
|
-
inject_into: request_parameter
|
515
|
-
field_name: "since_{{ config['cursor_field'] }}"
|
516
|
-
end_time_option:
|
517
|
-
type: RequestOption
|
518
|
-
inject_into: body_json
|
519
|
-
field_name: "before_{{ parameters['cursor_field'] }}"
|
520
|
-
partition_field_start: star
|
521
|
-
partition_field_end: en
|
522
|
-
"""
|
523
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
524
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
525
|
-
slicer_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["incremental"], {"cursor_field": "created_at"})
|
526
|
-
|
527
|
-
stream_slicer = factory.create_component(model_type=DatetimeBasedCursorModel, component_definition=slicer_manifest, config=input_config)
|
528
|
-
|
529
|
-
assert isinstance(stream_slicer, DatetimeBasedCursor)
|
530
|
-
assert stream_slicer._step == datetime.timedelta(days=10)
|
531
|
-
assert stream_slicer._cursor_field.string == "created"
|
532
|
-
assert stream_slicer.cursor_granularity == "PT0.000001S"
|
533
|
-
assert stream_slicer._lookback_window.string == "P5D"
|
534
|
-
assert stream_slicer.start_time_option.inject_into == RequestOptionType.request_parameter
|
535
|
-
assert stream_slicer.start_time_option.field_name.eval(config=input_config | {"cursor_field": "updated_at"}) == "since_updated_at"
|
536
|
-
assert stream_slicer.end_time_option.inject_into == RequestOptionType.body_json
|
537
|
-
assert stream_slicer.end_time_option.field_name.eval({}) == "before_created_at"
|
538
|
-
assert stream_slicer._partition_field_start.eval({}) == "star"
|
539
|
-
assert stream_slicer._partition_field_end.eval({}) == "en"
|
540
|
-
|
541
|
-
assert isinstance(stream_slicer._start_datetime, MinMaxDatetime)
|
542
|
-
assert stream_slicer.start_datetime._datetime_format == "%Y-%m-%dT%H:%M:%S.%f%z"
|
543
|
-
assert stream_slicer.start_datetime.datetime.string == "{{ config['start_time'] }}"
|
544
|
-
assert stream_slicer.start_datetime.min_datetime.string == "{{ config['start_time'] + day_delta(2) }}"
|
545
|
-
|
546
|
-
assert isinstance(stream_slicer._end_datetime, MinMaxDatetime)
|
547
|
-
assert stream_slicer._end_datetime.datetime.string == "{{ config['end_time'] }}"
|
548
|
-
|
549
|
-
|
550
|
-
def test_stream_with_incremental_and_retriever_with_partition_router():
|
551
|
-
content = """
|
552
|
-
decoder:
|
553
|
-
type: JsonDecoder
|
554
|
-
extractor:
|
555
|
-
type: DpathExtractor
|
556
|
-
decoder: "#/decoder"
|
557
|
-
selector:
|
558
|
-
type: RecordSelector
|
559
|
-
record_filter:
|
560
|
-
type: RecordFilter
|
561
|
-
condition: "{{ record['id'] > stream_state['id'] }}"
|
562
|
-
requester:
|
563
|
-
type: HttpRequester
|
564
|
-
name: "{{ parameters['name'] }}"
|
565
|
-
url_base: "https://api.sendgrid.com/v3/"
|
566
|
-
http_method: "GET"
|
567
|
-
authenticator:
|
568
|
-
type: BearerAuthenticator
|
569
|
-
api_token: "{{ config['apikey'] }}"
|
570
|
-
request_parameters:
|
571
|
-
unit: "day"
|
572
|
-
list_stream:
|
573
|
-
type: DeclarativeStream
|
574
|
-
schema_loader:
|
575
|
-
type: JsonFileSchemaLoader
|
576
|
-
file_path: "./source_sendgrid/schemas/{{ parameters.name }}.json"
|
577
|
-
incremental_sync:
|
578
|
-
type: DatetimeBasedCursor
|
579
|
-
$parameters:
|
580
|
-
datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
|
581
|
-
start_datetime: "{{ config['start_time'] }}"
|
582
|
-
end_datetime: "{{ config['end_time'] }}"
|
583
|
-
step: "P10D"
|
584
|
-
cursor_field: "created"
|
585
|
-
cursor_granularity: "PT0.000001S"
|
586
|
-
lookback_window: "P5D"
|
587
|
-
start_time_option:
|
588
|
-
inject_into: request_parameter
|
589
|
-
field_name: created[gte]
|
590
|
-
end_time_option:
|
591
|
-
inject_into: body_json
|
592
|
-
field_name: end_time
|
593
|
-
partition_field_start: star
|
594
|
-
partition_field_end: en
|
595
|
-
retriever:
|
596
|
-
type: SimpleRetriever
|
597
|
-
name: "{{ parameters['name'] }}"
|
598
|
-
partition_router:
|
599
|
-
type: ListPartitionRouter
|
600
|
-
values: "{{config['repos']}}"
|
601
|
-
cursor_field: a_key
|
602
|
-
request_option:
|
603
|
-
inject_into: header
|
604
|
-
field_name: a_key
|
605
|
-
paginator:
|
606
|
-
type: DefaultPaginator
|
607
|
-
page_size_option:
|
608
|
-
inject_into: request_parameter
|
609
|
-
field_name: page_size
|
610
|
-
page_token_option:
|
611
|
-
inject_into: path
|
612
|
-
type: RequestPath
|
613
|
-
pagination_strategy:
|
614
|
-
type: "CursorPagination"
|
615
|
-
cursor_value: "{{ response._metadata.next }}"
|
616
|
-
page_size: 10
|
617
|
-
requester:
|
618
|
-
$ref: "#/requester"
|
619
|
-
path: "{{ next_page_token['next_page_url'] }}"
|
620
|
-
record_selector:
|
621
|
-
$ref: "#/selector"
|
622
|
-
$parameters:
|
623
|
-
name: "lists"
|
624
|
-
primary_key: "id"
|
625
|
-
extractor:
|
626
|
-
$ref: "#/extractor"
|
627
|
-
field_path: ["{{ parameters['name'] }}"]
|
628
|
-
"""
|
629
|
-
|
630
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
631
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
632
|
-
stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["list_stream"], {})
|
633
|
-
|
634
|
-
stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
|
635
|
-
|
636
|
-
assert isinstance(stream, DeclarativeStream)
|
637
|
-
assert isinstance(stream.retriever, SimpleRetriever)
|
638
|
-
assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor)
|
639
|
-
|
640
|
-
datetime_stream_slicer = stream.retriever.stream_slicer._cursor_factory.create()
|
641
|
-
assert isinstance(datetime_stream_slicer, DatetimeBasedCursor)
|
642
|
-
assert isinstance(datetime_stream_slicer._start_datetime, MinMaxDatetime)
|
643
|
-
assert datetime_stream_slicer._start_datetime.datetime.string == "{{ config['start_time'] }}"
|
644
|
-
assert isinstance(datetime_stream_slicer._end_datetime, MinMaxDatetime)
|
645
|
-
assert datetime_stream_slicer._end_datetime.datetime.string == "{{ config['end_time'] }}"
|
646
|
-
assert datetime_stream_slicer.step == "P10D"
|
647
|
-
assert datetime_stream_slicer._cursor_field.string == "created"
|
648
|
-
|
649
|
-
list_stream_slicer = stream.retriever.stream_slicer._partition_router
|
650
|
-
assert isinstance(list_stream_slicer, ListPartitionRouter)
|
651
|
-
assert list_stream_slicer.values == ["airbyte", "airbyte-cloud"]
|
652
|
-
assert list_stream_slicer._cursor_field.string == "a_key"
|
653
|
-
|
654
|
-
|
655
|
-
def test_incremental_data_feed():
|
656
|
-
content = """
|
657
|
-
selector:
|
658
|
-
type: RecordSelector
|
659
|
-
extractor:
|
660
|
-
type: DpathExtractor
|
661
|
-
field_path: ["extractor_path"]
|
662
|
-
record_filter:
|
663
|
-
type: RecordFilter
|
664
|
-
condition: "{{ record['id'] > stream_state['id'] }}"
|
665
|
-
requester:
|
666
|
-
type: HttpRequester
|
667
|
-
name: "{{ parameters['name'] }}"
|
668
|
-
url_base: "https://api.sendgrid.com/v3/"
|
669
|
-
http_method: "GET"
|
670
|
-
list_stream:
|
671
|
-
type: DeclarativeStream
|
672
|
-
incremental_sync:
|
673
|
-
type: DatetimeBasedCursor
|
674
|
-
$parameters:
|
675
|
-
datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
|
676
|
-
start_datetime: "{{ config['start_time'] }}"
|
677
|
-
cursor_field: "created"
|
678
|
-
is_data_feed: true
|
679
|
-
retriever:
|
680
|
-
type: SimpleRetriever
|
681
|
-
name: "{{ parameters['name'] }}"
|
682
|
-
paginator:
|
683
|
-
type: DefaultPaginator
|
684
|
-
pagination_strategy:
|
685
|
-
type: "CursorPagination"
|
686
|
-
cursor_value: "{{ response._metadata.next }}"
|
687
|
-
page_size: 10
|
688
|
-
requester:
|
689
|
-
$ref: "#/requester"
|
690
|
-
path: "/"
|
691
|
-
record_selector:
|
692
|
-
$ref: "#/selector"
|
693
|
-
$parameters:
|
694
|
-
name: "lists"
|
695
|
-
"""
|
696
|
-
|
697
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
698
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
699
|
-
stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["list_stream"], {})
|
700
|
-
|
701
|
-
stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
|
702
|
-
|
703
|
-
assert isinstance(stream.retriever.paginator.pagination_strategy, StopConditionPaginationStrategyDecorator)
|
704
|
-
|
705
|
-
|
706
|
-
def test_given_data_feed_and_incremental_then_raise_error():
|
707
|
-
content = """
|
708
|
-
incremental_sync:
|
709
|
-
type: DatetimeBasedCursor
|
710
|
-
$parameters:
|
711
|
-
datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
|
712
|
-
start_datetime: "{{ config['start_time'] }}"
|
713
|
-
end_datetime: "2023-01-01"
|
714
|
-
cursor_field: "created"
|
715
|
-
is_data_feed: true"""
|
716
|
-
|
717
|
-
parsed_incremental_sync = YamlDeclarativeSource._parse(content)
|
718
|
-
resolved_incremental_sync = resolver.preprocess_manifest(parsed_incremental_sync)
|
719
|
-
datetime_based_cursor_definition = transformer.propagate_types_and_parameters("", resolved_incremental_sync["incremental_sync"], {})
|
720
|
-
|
721
|
-
with pytest.raises(ValueError):
|
722
|
-
factory.create_component(
|
723
|
-
model_type=DatetimeBasedCursorModel, component_definition=datetime_based_cursor_definition, config=input_config
|
724
|
-
)
|
725
|
-
|
726
|
-
|
727
|
-
@pytest.mark.parametrize(
|
728
|
-
"test_name, record_selector, expected_runtime_selector",
|
729
|
-
[("test_static_record_selector", "result", "result"), ("test_options_record_selector", "{{ parameters['name'] }}", "lists")],
|
730
|
-
)
|
731
|
-
def test_create_record_selector(test_name, record_selector, expected_runtime_selector):
|
732
|
-
content = f"""
|
733
|
-
extractor:
|
734
|
-
type: DpathExtractor
|
735
|
-
selector:
|
736
|
-
$parameters:
|
737
|
-
name: "lists"
|
738
|
-
type: RecordSelector
|
739
|
-
record_filter:
|
740
|
-
type: RecordFilter
|
741
|
-
condition: "{{{{ record['id'] > stream_state['id'] }}}}"
|
742
|
-
extractor:
|
743
|
-
$ref: "#/extractor"
|
744
|
-
field_path: ["{record_selector}"]
|
745
|
-
"""
|
746
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
747
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
748
|
-
selector_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["selector"], {})
|
749
|
-
|
750
|
-
selector = factory.create_component(
|
751
|
-
model_type=RecordSelectorModel, component_definition=selector_manifest, transformations=[], config=input_config
|
752
|
-
)
|
753
|
-
|
754
|
-
assert isinstance(selector, RecordSelector)
|
755
|
-
assert isinstance(selector.extractor, DpathExtractor)
|
756
|
-
assert [fp.eval(input_config) for fp in selector.extractor.field_path] == [expected_runtime_selector]
|
757
|
-
assert isinstance(selector.record_filter, RecordFilter)
|
758
|
-
assert selector.record_filter.condition == "{{ record['id'] > stream_state['id'] }}"
|
759
|
-
|
760
|
-
|
761
|
-
@pytest.mark.parametrize(
|
762
|
-
"test_name, error_handler, expected_backoff_strategy_type",
|
763
|
-
[
|
764
|
-
(
|
765
|
-
"test_create_requester_constant_error_handler",
|
766
|
-
"""
|
767
|
-
error_handler:
|
768
|
-
backoff_strategies:
|
769
|
-
- type: "ConstantBackoffStrategy"
|
770
|
-
backoff_time_in_seconds: 5
|
771
|
-
""",
|
772
|
-
ConstantBackoffStrategy,
|
773
|
-
),
|
774
|
-
(
|
775
|
-
"test_create_requester_exponential_error_handler",
|
776
|
-
"""
|
777
|
-
error_handler:
|
778
|
-
backoff_strategies:
|
779
|
-
- type: "ExponentialBackoffStrategy"
|
780
|
-
factor: 5
|
781
|
-
""",
|
782
|
-
ExponentialBackoffStrategy,
|
783
|
-
),
|
784
|
-
(
|
785
|
-
"test_create_requester_wait_time_from_header_error_handler",
|
786
|
-
"""
|
787
|
-
error_handler:
|
788
|
-
backoff_strategies:
|
789
|
-
- type: "WaitTimeFromHeader"
|
790
|
-
header: "a_header"
|
791
|
-
""",
|
792
|
-
WaitTimeFromHeaderBackoffStrategy,
|
793
|
-
),
|
794
|
-
(
|
795
|
-
"test_create_requester_wait_time_until_from_header_error_handler",
|
796
|
-
"""
|
797
|
-
error_handler:
|
798
|
-
backoff_strategies:
|
799
|
-
- type: "WaitUntilTimeFromHeader"
|
800
|
-
header: "a_header"
|
801
|
-
""",
|
802
|
-
WaitUntilTimeFromHeaderBackoffStrategy,
|
803
|
-
),
|
804
|
-
("test_create_requester_no_error_handler", """""", ExponentialBackoffStrategy),
|
805
|
-
],
|
806
|
-
)
|
807
|
-
def test_create_requester(test_name, error_handler, expected_backoff_strategy_type):
|
808
|
-
content = f"""
|
809
|
-
requester:
|
810
|
-
type: HttpRequester
|
811
|
-
path: "/v3/marketing/lists"
|
812
|
-
$parameters:
|
813
|
-
name: 'lists'
|
814
|
-
url_base: "https://api.sendgrid.com"
|
815
|
-
authenticator:
|
816
|
-
type: "BasicHttpAuthenticator"
|
817
|
-
username: "{{{{ parameters.name}}}}"
|
818
|
-
password: "{{{{ config.apikey }}}}"
|
819
|
-
request_parameters:
|
820
|
-
a_parameter: "something_here"
|
821
|
-
request_headers:
|
822
|
-
header: header_value
|
823
|
-
{error_handler}
|
824
|
-
"""
|
825
|
-
name = "name"
|
826
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
827
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
828
|
-
requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {})
|
829
|
-
|
830
|
-
selector = factory.create_component(
|
831
|
-
model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name
|
832
|
-
)
|
833
|
-
|
834
|
-
assert isinstance(selector, HttpRequester)
|
835
|
-
assert selector.http_method == HttpMethod.GET
|
836
|
-
assert selector.name == "name"
|
837
|
-
assert selector._path.string == "/v3/marketing/lists"
|
838
|
-
assert selector._url_base.string == "https://api.sendgrid.com"
|
839
|
-
|
840
|
-
assert isinstance(selector.error_handler, DefaultErrorHandler)
|
841
|
-
assert len(selector.error_handler.backoff_strategies) == 1
|
842
|
-
assert isinstance(selector.error_handler.backoff_strategies[0], expected_backoff_strategy_type)
|
843
|
-
|
844
|
-
assert isinstance(selector.authenticator, BasicHttpAuthenticator)
|
845
|
-
assert selector.authenticator._username.eval(input_config) == "lists"
|
846
|
-
assert selector.authenticator._password.eval(input_config) == "verysecrettoken"
|
847
|
-
|
848
|
-
assert isinstance(selector._request_options_provider, InterpolatedRequestOptionsProvider)
|
849
|
-
assert selector._request_options_provider._parameter_interpolator._interpolator.mapping["a_parameter"] == "something_here"
|
850
|
-
assert selector._request_options_provider._headers_interpolator._interpolator.mapping["header"] == "header_value"
|
851
|
-
|
852
|
-
|
853
|
-
def test_create_request_with_leacy_session_authenticator():
|
854
|
-
content = """
|
855
|
-
requester:
|
856
|
-
type: HttpRequester
|
857
|
-
path: "/v3/marketing/lists"
|
858
|
-
$parameters:
|
859
|
-
name: 'lists'
|
860
|
-
url_base: "https://api.sendgrid.com"
|
861
|
-
authenticator:
|
862
|
-
type: "LegacySessionTokenAuthenticator"
|
863
|
-
username: "{{ parameters.name}}"
|
864
|
-
password: "{{ config.apikey }}"
|
865
|
-
login_url: "login"
|
866
|
-
header: "token"
|
867
|
-
session_token_response_key: "session"
|
868
|
-
validate_session_url: validate
|
869
|
-
request_parameters:
|
870
|
-
a_parameter: "something_here"
|
871
|
-
request_headers:
|
872
|
-
header: header_value
|
873
|
-
"""
|
874
|
-
name = "name"
|
875
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
876
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
877
|
-
requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {})
|
878
|
-
|
879
|
-
selector = factory.create_component(
|
880
|
-
model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name
|
881
|
-
)
|
882
|
-
|
883
|
-
assert isinstance(selector, HttpRequester)
|
884
|
-
assert isinstance(selector.authenticator, LegacySessionTokenAuthenticator)
|
885
|
-
assert selector.authenticator._username.eval(input_config) == "lists"
|
886
|
-
assert selector.authenticator._password.eval(input_config) == "verysecrettoken"
|
887
|
-
assert selector.authenticator._api_url.eval(input_config) == "https://api.sendgrid.com"
|
888
|
-
|
889
|
-
|
890
|
-
def test_create_request_with_session_authenticator():
|
891
|
-
content = """
|
892
|
-
requester:
|
893
|
-
type: HttpRequester
|
894
|
-
path: "/v3/marketing/lists"
|
895
|
-
$parameters:
|
896
|
-
name: 'lists'
|
897
|
-
url_base: "https://api.sendgrid.com"
|
898
|
-
authenticator:
|
899
|
-
type: SessionTokenAuthenticator
|
900
|
-
expiration_duration: P10D
|
901
|
-
login_requester:
|
902
|
-
path: /session
|
903
|
-
type: HttpRequester
|
904
|
-
url_base: 'https://api.sendgrid.com'
|
905
|
-
http_method: POST
|
906
|
-
request_body_json:
|
907
|
-
password: '{{ config.apikey }}'
|
908
|
-
username: '{{ parameters.name }}'
|
909
|
-
session_token_path:
|
910
|
-
- id
|
911
|
-
request_authentication:
|
912
|
-
type: ApiKey
|
913
|
-
inject_into:
|
914
|
-
type: RequestOption
|
915
|
-
field_name: X-Metabase-Session
|
916
|
-
inject_into: header
|
917
|
-
request_parameters:
|
918
|
-
a_parameter: "something_here"
|
919
|
-
request_headers:
|
920
|
-
header: header_value
|
921
|
-
"""
|
922
|
-
name = "name"
|
923
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
924
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
925
|
-
requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {})
|
926
|
-
|
927
|
-
selector = factory.create_component(
|
928
|
-
model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name
|
929
|
-
)
|
930
|
-
|
931
|
-
assert isinstance(selector.authenticator, ApiKeyAuthenticator)
|
932
|
-
assert isinstance(selector.authenticator.token_provider, SessionTokenProvider)
|
933
|
-
assert selector.authenticator.token_provider.session_token_path == ["id"]
|
934
|
-
assert isinstance(selector.authenticator.token_provider.login_requester, HttpRequester)
|
935
|
-
assert selector.authenticator.token_provider.session_token_path == ["id"]
|
936
|
-
assert selector.authenticator.token_provider.login_requester._url_base.eval(input_config) == "https://api.sendgrid.com"
|
937
|
-
assert selector.authenticator.token_provider.login_requester.get_request_body_json() == {
|
938
|
-
"username": "lists",
|
939
|
-
"password": "verysecrettoken",
|
940
|
-
}
|
941
|
-
|
942
|
-
|
943
|
-
@pytest.mark.parametrize(
|
944
|
-
"input_config, expected_authenticator_class",
|
945
|
-
[
|
946
|
-
pytest.param(
|
947
|
-
{"auth": {"type": "token"}, "credentials": {"api_key": "some_key"}},
|
948
|
-
ApiKeyAuthenticator,
|
949
|
-
id="test_create_requester_with_selective_authenticator_and_token_selected",
|
950
|
-
),
|
951
|
-
pytest.param(
|
952
|
-
{"auth": {"type": "oauth"}, "credentials": {"client_id": "ABC"}},
|
953
|
-
DeclarativeOauth2Authenticator,
|
954
|
-
id="test_create_requester_with_selective_authenticator_and_oauth_selected",
|
955
|
-
),
|
956
|
-
],
|
957
|
-
)
|
958
|
-
def test_create_requester_with_selective_authenticator(input_config, expected_authenticator_class):
|
959
|
-
content = """
|
960
|
-
authenticator:
|
961
|
-
type: SelectiveAuthenticator
|
962
|
-
authenticator_selection_path:
|
963
|
-
- auth
|
964
|
-
- type
|
965
|
-
authenticators:
|
966
|
-
token:
|
967
|
-
type: ApiKeyAuthenticator
|
968
|
-
header: "Authorization"
|
969
|
-
api_token: "api_key={{ config['credentials']['api_key'] }}"
|
970
|
-
oauth:
|
971
|
-
type: OAuthAuthenticator
|
972
|
-
token_refresh_endpoint: https://api.url.com
|
973
|
-
client_id: "{{ config['credentials']['client_id'] }}"
|
974
|
-
client_secret: some_secret
|
975
|
-
refresh_token: some_token
|
976
|
-
"""
|
977
|
-
name = "name"
|
978
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
979
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
980
|
-
authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {})
|
981
|
-
|
982
|
-
authenticator = factory.create_component(
|
983
|
-
model_type=SelectiveAuthenticator, component_definition=authenticator_manifest, config=input_config, name=name
|
984
|
-
)
|
985
|
-
|
986
|
-
assert isinstance(authenticator, expected_authenticator_class)
|
987
|
-
|
988
|
-
|
989
|
-
def test_create_composite_error_handler():
|
990
|
-
content = """
|
991
|
-
error_handler:
|
992
|
-
type: "CompositeErrorHandler"
|
993
|
-
error_handlers:
|
994
|
-
- response_filters:
|
995
|
-
- predicate: "{{ 'code' in response }}"
|
996
|
-
action: RETRY
|
997
|
-
- response_filters:
|
998
|
-
- http_codes: [ 403 ]
|
999
|
-
action: RETRY
|
1000
|
-
"""
|
1001
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
1002
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
1003
|
-
error_handler_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["error_handler"], {})
|
1004
|
-
|
1005
|
-
error_handler = factory.create_component(
|
1006
|
-
model_type=CompositeErrorHandlerModel, component_definition=error_handler_manifest, config=input_config
|
1007
|
-
)
|
1008
|
-
|
1009
|
-
assert isinstance(error_handler, CompositeErrorHandler)
|
1010
|
-
assert len(error_handler.error_handlers) == 2
|
1011
|
-
|
1012
|
-
error_handler_0 = error_handler.error_handlers[0]
|
1013
|
-
assert isinstance(error_handler_0, DefaultErrorHandler)
|
1014
|
-
assert isinstance(error_handler_0.response_filters[0], HttpResponseFilter)
|
1015
|
-
assert error_handler_0.response_filters[0].predicate.condition == "{{ 'code' in response }}"
|
1016
|
-
assert error_handler_0.response_filters[0].action == ResponseAction.RETRY
|
1017
|
-
|
1018
|
-
error_handler_1 = error_handler.error_handlers[1]
|
1019
|
-
assert isinstance(error_handler_1, DefaultErrorHandler)
|
1020
|
-
assert isinstance(error_handler_1.response_filters[0], HttpResponseFilter)
|
1021
|
-
assert error_handler_1.response_filters[0].http_codes == {403}
|
1022
|
-
assert error_handler_1.response_filters[0].action == ResponseAction.RETRY
|
1023
|
-
|
1024
|
-
|
1025
|
-
# This might be a better test for the manifest transformer but also worth testing end-to-end here as well
|
1026
|
-
def test_config_with_defaults():
|
1027
|
-
content = """
|
1028
|
-
lists_stream:
|
1029
|
-
type: "DeclarativeStream"
|
1030
|
-
name: "lists"
|
1031
|
-
primary_key: id
|
1032
|
-
$parameters:
|
1033
|
-
name: "lists"
|
1034
|
-
url_base: "https://api.sendgrid.com"
|
1035
|
-
schema_loader:
|
1036
|
-
name: "{{ parameters.stream_name }}"
|
1037
|
-
file_path: "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
|
1038
|
-
retriever:
|
1039
|
-
paginator:
|
1040
|
-
type: "DefaultPaginator"
|
1041
|
-
page_size_option:
|
1042
|
-
type: RequestOption
|
1043
|
-
inject_into: request_parameter
|
1044
|
-
field_name: page_size
|
1045
|
-
page_token_option:
|
1046
|
-
type: RequestPath
|
1047
|
-
pagination_strategy:
|
1048
|
-
type: "CursorPagination"
|
1049
|
-
cursor_value: "{{ response._metadata.next }}"
|
1050
|
-
page_size: 10
|
1051
|
-
requester:
|
1052
|
-
path: "/v3/marketing/lists"
|
1053
|
-
authenticator:
|
1054
|
-
type: "BearerAuthenticator"
|
1055
|
-
api_token: "{{ config.apikey }}"
|
1056
|
-
request_parameters:
|
1057
|
-
page_size: 10
|
1058
|
-
record_selector:
|
1059
|
-
extractor:
|
1060
|
-
field_path: ["result"]
|
1061
|
-
streams:
|
1062
|
-
- "#/lists_stream"
|
1063
|
-
"""
|
1064
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
1065
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
1066
|
-
resolved_manifest["type"] = "DeclarativeSource"
|
1067
|
-
stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["lists_stream"], {})
|
1068
|
-
|
1069
|
-
stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
|
1070
|
-
|
1071
|
-
assert isinstance(stream, DeclarativeStream)
|
1072
|
-
assert stream.primary_key == "id"
|
1073
|
-
assert stream.name == "lists"
|
1074
|
-
assert isinstance(stream.retriever, SimpleRetriever)
|
1075
|
-
assert stream.retriever.name == stream.name
|
1076
|
-
assert stream.retriever.primary_key == stream.primary_key
|
1077
|
-
|
1078
|
-
assert isinstance(stream.schema_loader, JsonFileSchemaLoader)
|
1079
|
-
assert stream.schema_loader.file_path.string == "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
|
1080
|
-
assert stream.schema_loader.file_path.default == "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
|
1081
|
-
|
1082
|
-
assert isinstance(stream.retriever.requester, HttpRequester)
|
1083
|
-
assert stream.retriever.requester.http_method == HttpMethod.GET
|
1084
|
-
|
1085
|
-
assert isinstance(stream.retriever.requester.authenticator, BearerAuthenticator)
|
1086
|
-
assert stream.retriever.requester.authenticator.token_provider.get_token() == "verysecrettoken"
|
1087
|
-
|
1088
|
-
assert isinstance(stream.retriever.record_selector, RecordSelector)
|
1089
|
-
assert isinstance(stream.retriever.record_selector.extractor, DpathExtractor)
|
1090
|
-
assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_path] == ["result"]
|
1091
|
-
|
1092
|
-
assert isinstance(stream.retriever.paginator, DefaultPaginator)
|
1093
|
-
assert stream.retriever.paginator.url_base.string == "https://api.sendgrid.com"
|
1094
|
-
assert stream.retriever.paginator.pagination_strategy.get_page_size() == 10
|
1095
|
-
|
1096
|
-
|
1097
|
-
def test_create_default_paginator():
|
1098
|
-
content = """
|
1099
|
-
paginator:
|
1100
|
-
type: "DefaultPaginator"
|
1101
|
-
page_size_option:
|
1102
|
-
type: RequestOption
|
1103
|
-
inject_into: request_parameter
|
1104
|
-
field_name: page_size
|
1105
|
-
page_token_option:
|
1106
|
-
type: RequestPath
|
1107
|
-
pagination_strategy:
|
1108
|
-
type: "CursorPagination"
|
1109
|
-
page_size: 50
|
1110
|
-
cursor_value: "{{ response._metadata.next }}"
|
1111
|
-
"""
|
1112
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
1113
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
1114
|
-
paginator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["paginator"], {})
|
1115
|
-
|
1116
|
-
paginator = factory.create_component(
|
1117
|
-
model_type=DefaultPaginatorModel, component_definition=paginator_manifest, config=input_config, url_base="https://airbyte.io"
|
1118
|
-
)
|
1119
|
-
|
1120
|
-
assert isinstance(paginator, DefaultPaginator)
|
1121
|
-
assert paginator.url_base.string == "https://airbyte.io"
|
1122
|
-
|
1123
|
-
assert isinstance(paginator.pagination_strategy, CursorPaginationStrategy)
|
1124
|
-
assert paginator.pagination_strategy.page_size == 50
|
1125
|
-
assert paginator.pagination_strategy.cursor_value.string == "{{ response._metadata.next }}"
|
1126
|
-
|
1127
|
-
assert isinstance(paginator.page_size_option, RequestOption)
|
1128
|
-
assert paginator.page_size_option.inject_into == RequestOptionType.request_parameter
|
1129
|
-
assert paginator.page_size_option.field_name.eval(config=input_config) == "page_size"
|
1130
|
-
|
1131
|
-
assert isinstance(paginator.page_token_option, RequestPath)
|
1132
|
-
|
1133
|
-
|
1134
|
-
@pytest.mark.parametrize(
|
1135
|
-
"manifest, field_name, expected_value, expected_error",
|
1136
|
-
[
|
1137
|
-
pytest.param(
|
1138
|
-
{
|
1139
|
-
"type": "CustomErrorHandler",
|
1140
|
-
"class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
|
1141
|
-
"subcomponent_field_with_hint": {"type": "DpathExtractor", "field_path": []},
|
1142
|
-
},
|
1143
|
-
"subcomponent_field_with_hint",
|
1144
|
-
DpathExtractor(field_path=[], config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, parameters={}),
|
1145
|
-
None,
|
1146
|
-
id="test_create_custom_component_with_subcomponent_that_must_be_parsed",
|
1147
|
-
),
|
1148
|
-
pytest.param(
|
1149
|
-
{
|
1150
|
-
"type": "CustomErrorHandler",
|
1151
|
-
"class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
|
1152
|
-
"subcomponent_field_with_hint": {"field_path": []},
|
1153
|
-
},
|
1154
|
-
"subcomponent_field_with_hint",
|
1155
|
-
DpathExtractor(field_path=[], config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, parameters={}),
|
1156
|
-
None,
|
1157
|
-
id="test_create_custom_component_with_subcomponent_that_must_infer_type_from_explicit_hints",
|
1158
|
-
),
|
1159
|
-
pytest.param(
|
1160
|
-
{
|
1161
|
-
"type": "CustomErrorHandler",
|
1162
|
-
"class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
|
1163
|
-
"basic_field": "expected",
|
1164
|
-
},
|
1165
|
-
"basic_field",
|
1166
|
-
"expected",
|
1167
|
-
None,
|
1168
|
-
id="test_create_custom_component_with_built_in_type",
|
1169
|
-
),
|
1170
|
-
pytest.param(
|
1171
|
-
{
|
1172
|
-
"type": "CustomErrorHandler",
|
1173
|
-
"class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
|
1174
|
-
"optional_subcomponent_field": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "destination"},
|
1175
|
-
},
|
1176
|
-
"optional_subcomponent_field",
|
1177
|
-
RequestOption(inject_into=RequestOptionType.request_parameter, field_name="destination", parameters={}),
|
1178
|
-
None,
|
1179
|
-
id="test_create_custom_component_with_subcomponent_wrapped_in_optional",
|
1180
|
-
),
|
1181
|
-
pytest.param(
|
1182
|
-
{
|
1183
|
-
"type": "CustomErrorHandler",
|
1184
|
-
"class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
|
1185
|
-
"list_of_subcomponents": [
|
1186
|
-
{"inject_into": "header", "field_name": "store_me"},
|
1187
|
-
{"type": "RequestOption", "inject_into": "request_parameter", "field_name": "destination"},
|
1188
|
-
],
|
1189
|
-
},
|
1190
|
-
"list_of_subcomponents",
|
1191
|
-
[
|
1192
|
-
RequestOption(inject_into=RequestOptionType.header, field_name="store_me", parameters={}),
|
1193
|
-
RequestOption(inject_into=RequestOptionType.request_parameter, field_name="destination", parameters={}),
|
1194
|
-
],
|
1195
|
-
None,
|
1196
|
-
id="test_create_custom_component_with_subcomponent_wrapped_in_list",
|
1197
|
-
),
|
1198
|
-
pytest.param(
|
1199
|
-
{
|
1200
|
-
"type": "CustomErrorHandler",
|
1201
|
-
"class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
|
1202
|
-
"without_hint": {"inject_into": "request_parameter", "field_name": "missing_hint"},
|
1203
|
-
},
|
1204
|
-
"without_hint",
|
1205
|
-
None,
|
1206
|
-
None,
|
1207
|
-
id="test_create_custom_component_with_subcomponent_without_type_hints",
|
1208
|
-
),
|
1209
|
-
pytest.param(
|
1210
|
-
{
|
1211
|
-
"type": "CustomErrorHandler",
|
1212
|
-
"class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
|
1213
|
-
"paginator": {
|
1214
|
-
"type": "DefaultPaginator",
|
1215
|
-
"pagination_strategy": {"type": "OffsetIncrement", "page_size": 10},
|
1216
|
-
"$parameters": {"url_base": "https://physical_100.com"},
|
1217
|
-
},
|
1218
|
-
},
|
1219
|
-
"paginator",
|
1220
|
-
DefaultPaginator(
|
1221
|
-
pagination_strategy=OffsetIncrement(
|
1222
|
-
page_size=10, config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, parameters={}
|
1223
|
-
),
|
1224
|
-
url_base="https://physical_100.com",
|
1225
|
-
config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]},
|
1226
|
-
parameters={},
|
1227
|
-
),
|
1228
|
-
None,
|
1229
|
-
id="test_create_custom_component_with_subcomponent_that_uses_parameters",
|
1230
|
-
),
|
1231
|
-
pytest.param(
|
1232
|
-
{
|
1233
|
-
"type": "CustomErrorHandler",
|
1234
|
-
"class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingSomeComponent",
|
1235
|
-
"paginator": {
|
1236
|
-
"type": "DefaultPaginator",
|
1237
|
-
"pagination_strategy": {"type": "OffsetIncrement", "page_size": 10},
|
1238
|
-
},
|
1239
|
-
},
|
1240
|
-
"paginator",
|
1241
|
-
None,
|
1242
|
-
ValueError,
|
1243
|
-
id="test_create_custom_component_missing_required_field_emits_error",
|
1244
|
-
),
|
1245
|
-
pytest.param(
|
1246
|
-
{
|
1247
|
-
"type": "CustomErrorHandler",
|
1248
|
-
"class_name": "unit_tests.sources.declarative.parsers.testing_components.NonExistingClass",
|
1249
|
-
"paginator": {
|
1250
|
-
"type": "DefaultPaginator",
|
1251
|
-
"pagination_strategy": {"type": "OffsetIncrement", "page_size": 10},
|
1252
|
-
},
|
1253
|
-
},
|
1254
|
-
"paginator",
|
1255
|
-
None,
|
1256
|
-
ValueError,
|
1257
|
-
id="test_create_custom_component_non_existing_class_raises_value_error",
|
1258
|
-
),
|
1259
|
-
],
|
1260
|
-
)
|
1261
|
-
def test_create_custom_components(manifest, field_name, expected_value, expected_error):
|
1262
|
-
if expected_error:
|
1263
|
-
with pytest.raises(expected_error):
|
1264
|
-
factory.create_component(CustomErrorHandlerModel, manifest, input_config)
|
1265
|
-
else:
|
1266
|
-
custom_component = factory.create_component(CustomErrorHandlerModel, manifest, input_config)
|
1267
|
-
assert isinstance(custom_component, TestingSomeComponent)
|
1268
|
-
|
1269
|
-
assert isinstance(getattr(custom_component, field_name), type(expected_value))
|
1270
|
-
assert getattr(custom_component, field_name) == expected_value
|
1271
|
-
|
1272
|
-
|
1273
|
-
def test_custom_components_do_not_contain_extra_fields():
|
1274
|
-
custom_substream_partition_router_manifest = {
|
1275
|
-
"type": "CustomPartitionRouter",
|
1276
|
-
"class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingCustomSubstreamPartitionRouter",
|
1277
|
-
"custom_field": "here",
|
1278
|
-
"extra_field_to_exclude": "should_not_pass_as_parameter",
|
1279
|
-
"custom_pagination_strategy": {"type": "PageIncrement", "page_size": 100},
|
1280
|
-
"parent_stream_configs": [
|
1281
|
-
{
|
1282
|
-
"type": "ParentStreamConfig",
|
1283
|
-
"stream": {
|
1284
|
-
"type": "DeclarativeStream",
|
1285
|
-
"name": "a_parent",
|
1286
|
-
"primary_key": "id",
|
1287
|
-
"retriever": {
|
1288
|
-
"type": "SimpleRetriever",
|
1289
|
-
"record_selector": {
|
1290
|
-
"type": "RecordSelector",
|
1291
|
-
"extractor": {"type": "DpathExtractor", "field_path": []},
|
1292
|
-
},
|
1293
|
-
"requester": {"type": "HttpRequester", "url_base": "https://airbyte.io", "path": "some"},
|
1294
|
-
},
|
1295
|
-
"schema_loader": {
|
1296
|
-
"type": "JsonFileSchemaLoader",
|
1297
|
-
"file_path": "./source_sendgrid/schemas/{{ parameters['name'] }}.yaml",
|
1298
|
-
},
|
1299
|
-
},
|
1300
|
-
"parent_key": "id",
|
1301
|
-
"partition_field": "repository_id",
|
1302
|
-
"request_option": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "repository_id"},
|
1303
|
-
}
|
1304
|
-
],
|
1305
|
-
}
|
1306
|
-
|
1307
|
-
custom_substream_partition_router = factory.create_component(
|
1308
|
-
CustomPartitionRouterModel, custom_substream_partition_router_manifest, input_config
|
1309
|
-
)
|
1310
|
-
assert isinstance(custom_substream_partition_router, TestingCustomSubstreamPartitionRouter)
|
1311
|
-
|
1312
|
-
assert len(custom_substream_partition_router.parent_stream_configs) == 1
|
1313
|
-
assert custom_substream_partition_router.parent_stream_configs[0].parent_key.eval({}) == "id"
|
1314
|
-
assert custom_substream_partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id"
|
1315
|
-
assert custom_substream_partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter
|
1316
|
-
assert custom_substream_partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id"
|
1317
|
-
|
1318
|
-
assert isinstance(custom_substream_partition_router.custom_pagination_strategy, PageIncrement)
|
1319
|
-
assert custom_substream_partition_router.custom_pagination_strategy.page_size == 100
|
1320
|
-
|
1321
|
-
|
1322
|
-
def test_parse_custom_component_fields_if_subcomponent():
|
1323
|
-
custom_substream_partition_router_manifest = {
|
1324
|
-
"type": "CustomPartitionRouter",
|
1325
|
-
"class_name": "unit_tests.sources.declarative.parsers.testing_components.TestingCustomSubstreamPartitionRouter",
|
1326
|
-
"custom_field": "here",
|
1327
|
-
"custom_pagination_strategy": {"type": "PageIncrement", "page_size": 100},
|
1328
|
-
"parent_stream_configs": [
|
1329
|
-
{
|
1330
|
-
"type": "ParentStreamConfig",
|
1331
|
-
"stream": {
|
1332
|
-
"type": "DeclarativeStream",
|
1333
|
-
"name": "a_parent",
|
1334
|
-
"primary_key": "id",
|
1335
|
-
"retriever": {
|
1336
|
-
"type": "SimpleRetriever",
|
1337
|
-
"record_selector": {
|
1338
|
-
"type": "RecordSelector",
|
1339
|
-
"extractor": {"type": "DpathExtractor", "field_path": []},
|
1340
|
-
},
|
1341
|
-
"requester": {"type": "HttpRequester", "url_base": "https://airbyte.io", "path": "some"},
|
1342
|
-
},
|
1343
|
-
"schema_loader": {
|
1344
|
-
"type": "JsonFileSchemaLoader",
|
1345
|
-
"file_path": "./source_sendgrid/schemas/{{ parameters['name'] }}.yaml",
|
1346
|
-
},
|
1347
|
-
},
|
1348
|
-
"parent_key": "id",
|
1349
|
-
"partition_field": "repository_id",
|
1350
|
-
"request_option": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "repository_id"},
|
1351
|
-
}
|
1352
|
-
],
|
1353
|
-
}
|
1354
|
-
|
1355
|
-
custom_substream_partition_router = factory.create_component(
|
1356
|
-
CustomPartitionRouterModel, custom_substream_partition_router_manifest, input_config
|
1357
|
-
)
|
1358
|
-
assert isinstance(custom_substream_partition_router, TestingCustomSubstreamPartitionRouter)
|
1359
|
-
assert custom_substream_partition_router.custom_field == "here"
|
1360
|
-
|
1361
|
-
assert len(custom_substream_partition_router.parent_stream_configs) == 1
|
1362
|
-
assert custom_substream_partition_router.parent_stream_configs[0].parent_key.eval({}) == "id"
|
1363
|
-
assert custom_substream_partition_router.parent_stream_configs[0].partition_field.eval({}) == "repository_id"
|
1364
|
-
assert custom_substream_partition_router.parent_stream_configs[0].request_option.inject_into == RequestOptionType.request_parameter
|
1365
|
-
assert custom_substream_partition_router.parent_stream_configs[0].request_option.field_name.eval(config=input_config) == "repository_id"
|
1366
|
-
|
1367
|
-
assert isinstance(custom_substream_partition_router.custom_pagination_strategy, PageIncrement)
|
1368
|
-
assert custom_substream_partition_router.custom_pagination_strategy.page_size == 100
|
1369
|
-
|
1370
|
-
|
1371
|
-
class TestCreateTransformations:
|
1372
|
-
# the tabbing matters
|
1373
|
-
base_parameters = """
|
1374
|
-
name: "lists"
|
1375
|
-
primary_key: id
|
1376
|
-
url_base: "https://api.sendgrid.com"
|
1377
|
-
schema_loader:
|
1378
|
-
name: "{{ parameters.name }}"
|
1379
|
-
file_path: "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
|
1380
|
-
retriever:
|
1381
|
-
requester:
|
1382
|
-
name: "{{ parameters.name }}"
|
1383
|
-
path: "/v3/marketing/lists"
|
1384
|
-
request_parameters:
|
1385
|
-
page_size: 10
|
1386
|
-
record_selector:
|
1387
|
-
extractor:
|
1388
|
-
field_path: ["result"]
|
1389
|
-
"""
|
1390
|
-
|
1391
|
-
def test_no_transformations(self):
|
1392
|
-
content = f"""
|
1393
|
-
the_stream:
|
1394
|
-
type: DeclarativeStream
|
1395
|
-
$parameters:
|
1396
|
-
{self.base_parameters}
|
1397
|
-
"""
|
1398
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
1399
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
1400
|
-
resolved_manifest["type"] = "DeclarativeSource"
|
1401
|
-
stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["the_stream"], {})
|
1402
|
-
|
1403
|
-
stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
|
1404
|
-
|
1405
|
-
assert isinstance(stream, DeclarativeStream)
|
1406
|
-
assert [] == stream.retriever.record_selector.transformations
|
1407
|
-
|
1408
|
-
def test_remove_fields(self):
|
1409
|
-
content = f"""
|
1410
|
-
the_stream:
|
1411
|
-
type: DeclarativeStream
|
1412
|
-
$parameters:
|
1413
|
-
{self.base_parameters}
|
1414
|
-
transformations:
|
1415
|
-
- type: RemoveFields
|
1416
|
-
field_pointers:
|
1417
|
-
- ["path", "to", "field1"]
|
1418
|
-
- ["path2"]
|
1419
|
-
"""
|
1420
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
1421
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
1422
|
-
resolved_manifest["type"] = "DeclarativeSource"
|
1423
|
-
stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["the_stream"], {})
|
1424
|
-
|
1425
|
-
stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
|
1426
|
-
|
1427
|
-
assert isinstance(stream, DeclarativeStream)
|
1428
|
-
expected = [RemoveFields(field_pointers=[["path", "to", "field1"], ["path2"]], parameters={})]
|
1429
|
-
assert stream.retriever.record_selector.transformations == expected
|
1430
|
-
|
1431
|
-
def test_add_fields_no_value_type(self):
|
1432
|
-
content = f"""
|
1433
|
-
the_stream:
|
1434
|
-
type: DeclarativeStream
|
1435
|
-
$parameters:
|
1436
|
-
{self.base_parameters}
|
1437
|
-
transformations:
|
1438
|
-
- type: AddFields
|
1439
|
-
fields:
|
1440
|
-
- path: ["field1"]
|
1441
|
-
value: "static_value"
|
1442
|
-
"""
|
1443
|
-
expected = [
|
1444
|
-
AddFields(
|
1445
|
-
fields=[
|
1446
|
-
AddedFieldDefinition(
|
1447
|
-
path=["field1"],
|
1448
|
-
value=InterpolatedString(string="static_value", default="static_value", parameters={}),
|
1449
|
-
value_type=None,
|
1450
|
-
parameters={},
|
1451
|
-
)
|
1452
|
-
],
|
1453
|
-
parameters={},
|
1454
|
-
)
|
1455
|
-
]
|
1456
|
-
self._test_add_fields(content, expected)
|
1457
|
-
|
1458
|
-
def test_add_fields_value_type_is_string(self):
|
1459
|
-
content = f"""
|
1460
|
-
the_stream:
|
1461
|
-
type: DeclarativeStream
|
1462
|
-
$parameters:
|
1463
|
-
{self.base_parameters}
|
1464
|
-
transformations:
|
1465
|
-
- type: AddFields
|
1466
|
-
fields:
|
1467
|
-
- path: ["field1"]
|
1468
|
-
value: "static_value"
|
1469
|
-
value_type: string
|
1470
|
-
"""
|
1471
|
-
expected = [
|
1472
|
-
AddFields(
|
1473
|
-
fields=[
|
1474
|
-
AddedFieldDefinition(
|
1475
|
-
path=["field1"],
|
1476
|
-
value=InterpolatedString(string="static_value", default="static_value", parameters={}),
|
1477
|
-
value_type=str,
|
1478
|
-
parameters={},
|
1479
|
-
)
|
1480
|
-
],
|
1481
|
-
parameters={},
|
1482
|
-
)
|
1483
|
-
]
|
1484
|
-
self._test_add_fields(content, expected)
|
1485
|
-
|
1486
|
-
def test_add_fields_value_type_is_number(self):
|
1487
|
-
content = f"""
|
1488
|
-
the_stream:
|
1489
|
-
type: DeclarativeStream
|
1490
|
-
$parameters:
|
1491
|
-
{self.base_parameters}
|
1492
|
-
transformations:
|
1493
|
-
- type: AddFields
|
1494
|
-
fields:
|
1495
|
-
- path: ["field1"]
|
1496
|
-
value: "1"
|
1497
|
-
value_type: number
|
1498
|
-
"""
|
1499
|
-
expected = [
|
1500
|
-
AddFields(
|
1501
|
-
fields=[
|
1502
|
-
AddedFieldDefinition(
|
1503
|
-
path=["field1"],
|
1504
|
-
value=InterpolatedString(string="1", default="1", parameters={}),
|
1505
|
-
value_type=float,
|
1506
|
-
parameters={},
|
1507
|
-
)
|
1508
|
-
],
|
1509
|
-
parameters={},
|
1510
|
-
)
|
1511
|
-
]
|
1512
|
-
self._test_add_fields(content, expected)
|
1513
|
-
|
1514
|
-
def test_add_fields_value_type_is_integer(self):
|
1515
|
-
content = f"""
|
1516
|
-
the_stream:
|
1517
|
-
type: DeclarativeStream
|
1518
|
-
$parameters:
|
1519
|
-
{self.base_parameters}
|
1520
|
-
transformations:
|
1521
|
-
- type: AddFields
|
1522
|
-
fields:
|
1523
|
-
- path: ["field1"]
|
1524
|
-
value: "1"
|
1525
|
-
value_type: integer
|
1526
|
-
"""
|
1527
|
-
expected = [
|
1528
|
-
AddFields(
|
1529
|
-
fields=[
|
1530
|
-
AddedFieldDefinition(
|
1531
|
-
path=["field1"],
|
1532
|
-
value=InterpolatedString(string="1", default="1", parameters={}),
|
1533
|
-
value_type=int,
|
1534
|
-
parameters={},
|
1535
|
-
)
|
1536
|
-
],
|
1537
|
-
parameters={},
|
1538
|
-
)
|
1539
|
-
]
|
1540
|
-
self._test_add_fields(content, expected)
|
1541
|
-
|
1542
|
-
def test_add_fields_value_type_is_boolean(self):
|
1543
|
-
content = f"""
|
1544
|
-
the_stream:
|
1545
|
-
type: DeclarativeStream
|
1546
|
-
$parameters:
|
1547
|
-
{self.base_parameters}
|
1548
|
-
transformations:
|
1549
|
-
- type: AddFields
|
1550
|
-
fields:
|
1551
|
-
- path: ["field1"]
|
1552
|
-
value: False
|
1553
|
-
value_type: boolean
|
1554
|
-
"""
|
1555
|
-
expected = [
|
1556
|
-
AddFields(
|
1557
|
-
fields=[
|
1558
|
-
AddedFieldDefinition(
|
1559
|
-
path=["field1"],
|
1560
|
-
value=InterpolatedString(string="False", default="False", parameters={}),
|
1561
|
-
value_type=bool,
|
1562
|
-
parameters={},
|
1563
|
-
)
|
1564
|
-
],
|
1565
|
-
parameters={},
|
1566
|
-
)
|
1567
|
-
]
|
1568
|
-
self._test_add_fields(content, expected)
|
1569
|
-
|
1570
|
-
def _test_add_fields(self, content, expected):
|
1571
|
-
parsed_manifest = YamlDeclarativeSource._parse(content)
|
1572
|
-
resolved_manifest = resolver.preprocess_manifest(parsed_manifest)
|
1573
|
-
resolved_manifest["type"] = "DeclarativeSource"
|
1574
|
-
stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["the_stream"], {})
|
1575
|
-
|
1576
|
-
stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config)
|
1577
|
-
|
1578
|
-
assert isinstance(stream, DeclarativeStream)
|
1579
|
-
assert stream.retriever.record_selector.transformations == expected
|
1580
|
-
|
1581
|
-
def test_default_schema_loader(self):
|
1582
|
-
component_definition = {
|
1583
|
-
"type": "DeclarativeStream",
|
1584
|
-
"name": "test",
|
1585
|
-
"primary_key": [],
|
1586
|
-
"retriever": {
|
1587
|
-
"type": "SimpleRetriever",
|
1588
|
-
"requester": {
|
1589
|
-
"type": "HttpRequester",
|
1590
|
-
"url_base": "http://localhost:6767/",
|
1591
|
-
"path": "items/",
|
1592
|
-
"request_options_provider": {
|
1593
|
-
"request_parameters": {},
|
1594
|
-
"request_headers": {},
|
1595
|
-
"request_body_json": {},
|
1596
|
-
"type": "InterpolatedRequestOptionsProvider",
|
1597
|
-
},
|
1598
|
-
"authenticator": {"type": "BearerAuthenticator", "api_token": "{{ config['api_key'] }}"},
|
1599
|
-
},
|
1600
|
-
"record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": ["items"]}},
|
1601
|
-
"paginator": {"type": "NoPagination"},
|
1602
|
-
},
|
1603
|
-
}
|
1604
|
-
resolved_manifest = resolver.preprocess_manifest(component_definition)
|
1605
|
-
ws = ManifestComponentTransformer()
|
1606
|
-
propagated_source_config = ws.propagate_types_and_parameters("", resolved_manifest, {})
|
1607
|
-
stream = factory.create_component(
|
1608
|
-
model_type=DeclarativeStreamModel, component_definition=propagated_source_config, config=input_config
|
1609
|
-
)
|
1610
|
-
schema_loader = stream.schema_loader
|
1611
|
-
assert schema_loader.default_loader._get_json_filepath().split("/")[-1] == f"{stream.name}.json"
|
1612
|
-
|
1613
|
-
|
1614
|
-
@pytest.mark.parametrize(
|
1615
|
-
"incremental, partition_router, expected_type",
|
1616
|
-
[
|
1617
|
-
pytest.param(
|
1618
|
-
{
|
1619
|
-
"type": "DatetimeBasedCursor",
|
1620
|
-
"datetime_format": "%Y-%m-%dT%H:%M:%S.%f%z",
|
1621
|
-
"start_datetime": "{{ config['start_time'] }}",
|
1622
|
-
"end_datetime": "{{ config['end_time'] }}",
|
1623
|
-
"step": "P10D",
|
1624
|
-
"cursor_field": "created",
|
1625
|
-
"cursor_granularity": "PT0.000001S",
|
1626
|
-
},
|
1627
|
-
None,
|
1628
|
-
DatetimeBasedCursor,
|
1629
|
-
id="test_create_simple_retriever_with_incremental",
|
1630
|
-
),
|
1631
|
-
pytest.param(
|
1632
|
-
None,
|
1633
|
-
{
|
1634
|
-
"type": "ListPartitionRouter",
|
1635
|
-
"values": "{{config['repos']}}",
|
1636
|
-
"cursor_field": "a_key",
|
1637
|
-
},
|
1638
|
-
ListPartitionRouter,
|
1639
|
-
id="test_create_simple_retriever_with_partition_router",
|
1640
|
-
),
|
1641
|
-
pytest.param(
|
1642
|
-
{
|
1643
|
-
"type": "DatetimeBasedCursor",
|
1644
|
-
"datetime_format": "%Y-%m-%dT%H:%M:%S.%f%z",
|
1645
|
-
"start_datetime": "{{ config['start_time'] }}",
|
1646
|
-
"end_datetime": "{{ config['end_time'] }}",
|
1647
|
-
"step": "P10D",
|
1648
|
-
"cursor_field": "created",
|
1649
|
-
"cursor_granularity": "PT0.000001S",
|
1650
|
-
},
|
1651
|
-
{
|
1652
|
-
"type": "ListPartitionRouter",
|
1653
|
-
"values": "{{config['repos']}}",
|
1654
|
-
"cursor_field": "a_key",
|
1655
|
-
},
|
1656
|
-
PerPartitionCursor,
|
1657
|
-
id="test_create_simple_retriever_with_incremental_and_partition_router",
|
1658
|
-
),
|
1659
|
-
pytest.param(
|
1660
|
-
{
|
1661
|
-
"type": "DatetimeBasedCursor",
|
1662
|
-
"datetime_format": "%Y-%m-%dT%H:%M:%S.%f%z",
|
1663
|
-
"start_datetime": "{{ config['start_time'] }}",
|
1664
|
-
"end_datetime": "{{ config['end_time'] }}",
|
1665
|
-
"step": "P10D",
|
1666
|
-
"cursor_field": "created",
|
1667
|
-
"cursor_granularity": "PT0.000001S",
|
1668
|
-
},
|
1669
|
-
[
|
1670
|
-
{
|
1671
|
-
"type": "ListPartitionRouter",
|
1672
|
-
"values": "{{config['repos']}}",
|
1673
|
-
"cursor_field": "a_key",
|
1674
|
-
},
|
1675
|
-
{
|
1676
|
-
"type": "ListPartitionRouter",
|
1677
|
-
"values": "{{config['repos']}}",
|
1678
|
-
"cursor_field": "b_key",
|
1679
|
-
},
|
1680
|
-
],
|
1681
|
-
PerPartitionCursor,
|
1682
|
-
id="test_create_simple_retriever_with_partition_routers_multiple_components",
|
1683
|
-
),
|
1684
|
-
pytest.param(None, None, SinglePartitionRouter, id="test_create_simple_retriever_with_no_incremental_or_partition_router"),
|
1685
|
-
],
|
1686
|
-
)
|
1687
|
-
def test_merge_incremental_and_partition_router(incremental, partition_router, expected_type):
|
1688
|
-
stream_model = {
|
1689
|
-
"type": "DeclarativeStream",
|
1690
|
-
"retriever": {
|
1691
|
-
"type": "SimpleRetriever",
|
1692
|
-
"record_selector": {
|
1693
|
-
"type": "RecordSelector",
|
1694
|
-
"extractor": {
|
1695
|
-
"type": "DpathExtractor",
|
1696
|
-
"field_path": [],
|
1697
|
-
},
|
1698
|
-
},
|
1699
|
-
"requester": {
|
1700
|
-
"type": "HttpRequester",
|
1701
|
-
"name": "list",
|
1702
|
-
"url_base": "orange.com",
|
1703
|
-
"path": "/v1/api",
|
1704
|
-
},
|
1705
|
-
},
|
1706
|
-
}
|
1707
|
-
|
1708
|
-
if incremental:
|
1709
|
-
stream_model["incremental_sync"] = incremental
|
1710
|
-
|
1711
|
-
if partition_router:
|
1712
|
-
stream_model["retriever"]["partition_router"] = partition_router
|
1713
|
-
|
1714
|
-
stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_model, config=input_config)
|
1715
|
-
|
1716
|
-
assert isinstance(stream, DeclarativeStream)
|
1717
|
-
assert isinstance(stream.retriever, SimpleRetriever)
|
1718
|
-
assert isinstance(stream.retriever.stream_slicer, expected_type)
|
1719
|
-
|
1720
|
-
if incremental and partition_router:
|
1721
|
-
assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor)
|
1722
|
-
if type(partition_router) == list and len(partition_router) > 1:
|
1723
|
-
assert type(stream.retriever.stream_slicer._partition_router) == CartesianProductStreamSlicer
|
1724
|
-
assert len(stream.retriever.stream_slicer._partition_router.stream_slicers) == len(partition_router)
|
1725
|
-
elif partition_router and type(partition_router) == list and len(partition_router) > 1:
|
1726
|
-
assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor)
|
1727
|
-
assert len(stream.retriever.stream_slicer.stream_slicerS) == len(partition_router)
|
1728
|
-
|
1729
|
-
|
1730
|
-
def test_simple_retriever_emit_log_messages():
|
1731
|
-
simple_retriever_model = {
|
1732
|
-
"type": "SimpleRetriever",
|
1733
|
-
"record_selector": {
|
1734
|
-
"type": "RecordSelector",
|
1735
|
-
"extractor": {
|
1736
|
-
"type": "DpathExtractor",
|
1737
|
-
"field_path": [],
|
1738
|
-
},
|
1739
|
-
},
|
1740
|
-
"requester": {"type": "HttpRequester", "name": "list", "url_base": "orange.com", "path": "/v1/api"},
|
1741
|
-
}
|
1742
|
-
|
1743
|
-
connector_builder_factory = ModelToComponentFactory(emit_connector_builder_messages=True)
|
1744
|
-
retriever = connector_builder_factory.create_component(
|
1745
|
-
model_type=SimpleRetrieverModel,
|
1746
|
-
component_definition=simple_retriever_model,
|
1747
|
-
config={},
|
1748
|
-
name="Test",
|
1749
|
-
primary_key="id",
|
1750
|
-
stream_slicer=None,
|
1751
|
-
transformations=[],
|
1752
|
-
)
|
1753
|
-
|
1754
|
-
assert isinstance(retriever, SimpleRetrieverTestReadDecorator)
|
1755
|
-
assert connector_builder_factory._message_repository._log_level == Level.DEBUG
|
1756
|
-
|
1757
|
-
|
1758
|
-
def test_ignore_retry():
|
1759
|
-
requester_model = {
|
1760
|
-
"type": "HttpRequester",
|
1761
|
-
"name": "list",
|
1762
|
-
"url_base": "orange.com",
|
1763
|
-
"path": "/v1/api",
|
1764
|
-
}
|
1765
|
-
|
1766
|
-
connector_builder_factory = ModelToComponentFactory(disable_retries=True)
|
1767
|
-
requester = connector_builder_factory.create_component(
|
1768
|
-
model_type=HttpRequesterModel,
|
1769
|
-
component_definition=requester_model,
|
1770
|
-
config={},
|
1771
|
-
name="Test",
|
1772
|
-
)
|
1773
|
-
|
1774
|
-
assert requester.max_retries == 0
|
1775
|
-
|
1776
|
-
|
1777
|
-
def test_create_page_increment():
|
1778
|
-
model = PageIncrementModel(
|
1779
|
-
type="PageIncrement",
|
1780
|
-
page_size=10,
|
1781
|
-
start_from_page=1,
|
1782
|
-
inject_on_first_request=True,
|
1783
|
-
)
|
1784
|
-
expected_strategy = PageIncrement(page_size=10, start_from_page=1, inject_on_first_request=True, parameters={}, config=input_config)
|
1785
|
-
|
1786
|
-
strategy = factory.create_page_increment(model, input_config)
|
1787
|
-
|
1788
|
-
assert strategy.page_size == expected_strategy.page_size
|
1789
|
-
assert strategy.start_from_page == expected_strategy.start_from_page
|
1790
|
-
assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request
|
1791
|
-
|
1792
|
-
|
1793
|
-
def test_create_page_increment_with_interpolated_page_size():
|
1794
|
-
model = PageIncrementModel(
|
1795
|
-
type="PageIncrement",
|
1796
|
-
page_size="{{ config['page_size'] }}",
|
1797
|
-
start_from_page=1,
|
1798
|
-
inject_on_first_request=True,
|
1799
|
-
)
|
1800
|
-
config = {
|
1801
|
-
**input_config,
|
1802
|
-
"page_size": 5
|
1803
|
-
}
|
1804
|
-
expected_strategy = PageIncrement(page_size=5, start_from_page=1, inject_on_first_request=True, parameters={}, config=config)
|
1805
|
-
|
1806
|
-
strategy = factory.create_page_increment(model, config)
|
1807
|
-
|
1808
|
-
assert strategy.get_page_size() == expected_strategy.get_page_size()
|
1809
|
-
assert strategy.start_from_page == expected_strategy.start_from_page
|
1810
|
-
assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request
|
1811
|
-
|
1812
|
-
|
1813
|
-
def test_create_offset_increment():
|
1814
|
-
model = OffsetIncrementModel(
|
1815
|
-
type="OffsetIncrement",
|
1816
|
-
page_size=10,
|
1817
|
-
inject_on_first_request=True,
|
1818
|
-
)
|
1819
|
-
expected_strategy = OffsetIncrement(page_size=10, inject_on_first_request=True, parameters={}, config=input_config)
|
1820
|
-
|
1821
|
-
strategy = factory.create_offset_increment(model, input_config)
|
1822
|
-
|
1823
|
-
assert strategy.page_size == expected_strategy.page_size
|
1824
|
-
assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request
|
1825
|
-
assert strategy.config == input_config
|
1826
|
-
|
1827
|
-
|
1828
|
-
class MyCustomSchemaLoader(SchemaLoader):
|
1829
|
-
def get_json_schema(self) -> Mapping[str, Any]:
|
1830
|
-
"""Returns a mapping describing the stream's schema"""
|
1831
|
-
return {}
|
1832
|
-
|
1833
|
-
|
1834
|
-
def test_create_custom_schema_loader():
|
1835
|
-
|
1836
|
-
definition = {
|
1837
|
-
"type": "CustomSchemaLoader",
|
1838
|
-
"class_name": "unit_tests.sources.declarative.parsers.test_model_to_component_factory.MyCustomSchemaLoader"
|
1839
|
-
}
|
1840
|
-
component = factory.create_component(CustomSchemaLoaderModel, definition, {})
|
1841
|
-
assert isinstance(component, MyCustomSchemaLoader)
|