airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- airbyte_cdk/__init__.py +355 -6
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +29 -10
- airbyte_cdk/connector.py +24 -24
- airbyte_cdk/connector_builder/README.md +53 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
- airbyte_cdk/connector_builder/main.py +45 -13
- airbyte_cdk/connector_builder/message_grouper.py +189 -50
- airbyte_cdk/connector_builder/models.py +3 -2
- airbyte_cdk/destinations/__init__.py +4 -3
- airbyte_cdk/destinations/destination.py +54 -20
- airbyte_cdk/destinations/vector_db_based/README.md +37 -0
- airbyte_cdk/destinations/vector_db_based/config.py +40 -17
- airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
- airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
- airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
- airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
- airbyte_cdk/entrypoint.py +153 -44
- airbyte_cdk/exception_handler.py +21 -3
- airbyte_cdk/logger.py +30 -44
- airbyte_cdk/models/__init__.py +13 -2
- airbyte_cdk/models/airbyte_protocol.py +86 -1
- airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
- airbyte_cdk/models/file_transfer_record_message.py +13 -0
- airbyte_cdk/models/well_known_types.py +1 -1
- airbyte_cdk/sources/__init__.py +5 -1
- airbyte_cdk/sources/abstract_source.py +125 -79
- airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
- airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
- airbyte_cdk/sources/config.py +3 -2
- airbyte_cdk/sources/connector_state_manager.py +49 -83
- airbyte_cdk/sources/declarative/async_job/job.py +52 -0
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
- airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
- airbyte_cdk/sources/declarative/async_job/status.py +24 -0
- airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
- airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
- airbyte_cdk/sources/declarative/auth/token.py +28 -10
- airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
- airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
- airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
- airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1213 -88
- airbyte_cdk/sources/declarative/declarative_source.py +5 -2
- airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
- airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
- airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
- airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
- airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
- airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
- airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
- airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
- airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
- airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +346 -0
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
- airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +173 -74
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
- airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
- airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
- airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
- airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
- airbyte_cdk/sources/declarative/models/__init__.py +1 -1
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1329 -595
- airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1763 -226
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
- airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
- airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
- airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
- airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
- airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
- airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
- airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
- airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
- airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
- airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
- airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
- airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
- airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
- airbyte_cdk/sources/declarative/spec/spec.py +12 -5
- airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
- airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
- airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
- airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
- airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
- airbyte_cdk/sources/declarative/types.py +19 -110
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
- airbyte_cdk/sources/embedded/base_integration.py +16 -5
- airbyte_cdk/sources/embedded/catalog.py +16 -4
- airbyte_cdk/sources/embedded/runner.py +19 -3
- airbyte_cdk/sources/embedded/tools.py +5 -2
- airbyte_cdk/sources/file_based/README.md +152 -0
- airbyte_cdk/sources/file_based/__init__.py +24 -0
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
- airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
- airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
- airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
- airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
- airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
- airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
- airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
- airbyte_cdk/sources/file_based/exceptions.py +18 -15
- airbyte_cdk/sources/file_based/file_based_source.py +140 -33
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
- airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
- airbyte_cdk/sources/file_based/remote_file.py +1 -1
- airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
- airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
- airbyte_cdk/sources/http_logger.py +8 -3
- airbyte_cdk/sources/message/__init__.py +7 -1
- airbyte_cdk/sources/message/repository.py +18 -4
- airbyte_cdk/sources/source.py +42 -38
- airbyte_cdk/sources/streams/__init__.py +2 -2
- airbyte_cdk/sources/streams/availability_strategy.py +54 -3
- airbyte_cdk/sources/streams/call_rate.py +64 -21
- airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
- airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
- airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
- airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
- airbyte_cdk/sources/streams/concurrent/README.md +7 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
- airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
- airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
- airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
- airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
- airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
- airbyte_cdk/sources/streams/core.py +412 -87
- airbyte_cdk/sources/streams/http/__init__.py +2 -1
- airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
- airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
- airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
- airbyte_cdk/sources/streams/http/exceptions.py +27 -7
- airbyte_cdk/sources/streams/http/http.py +369 -246
- airbyte_cdk/sources/streams/http/http_client.py +531 -0
- airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
- airbyte_cdk/sources/types.py +154 -0
- airbyte_cdk/sources/utils/record_helper.py +36 -21
- airbyte_cdk/sources/utils/schema_helpers.py +13 -6
- airbyte_cdk/sources/utils/slice_logger.py +4 -1
- airbyte_cdk/sources/utils/transform.py +54 -20
- airbyte_cdk/sql/_util/hashing.py +34 -0
- airbyte_cdk/sql/_util/name_normalizers.py +92 -0
- airbyte_cdk/sql/constants.py +32 -0
- airbyte_cdk/sql/exceptions.py +235 -0
- airbyte_cdk/sql/secrets.py +123 -0
- airbyte_cdk/sql/shared/__init__.py +15 -0
- airbyte_cdk/sql/shared/catalog_providers.py +145 -0
- airbyte_cdk/sql/shared/sql_processor.py +786 -0
- airbyte_cdk/sql/types.py +160 -0
- airbyte_cdk/test/catalog_builder.py +70 -18
- airbyte_cdk/test/entrypoint_wrapper.py +117 -42
- airbyte_cdk/test/mock_http/__init__.py +1 -1
- airbyte_cdk/test/mock_http/matcher.py +6 -0
- airbyte_cdk/test/mock_http/mocker.py +57 -10
- airbyte_cdk/test/mock_http/request.py +19 -3
- airbyte_cdk/test/mock_http/response.py +3 -1
- airbyte_cdk/test/mock_http/response_builder.py +32 -16
- airbyte_cdk/test/state_builder.py +18 -10
- airbyte_cdk/test/utils/__init__.py +1 -0
- airbyte_cdk/test/utils/data.py +24 -0
- airbyte_cdk/test/utils/http_mocking.py +16 -0
- airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
- airbyte_cdk/test/utils/reading.py +26 -0
- airbyte_cdk/utils/__init__.py +2 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
- airbyte_cdk/utils/analytics_message.py +10 -2
- airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
- airbyte_cdk/utils/event_timing.py +10 -10
- airbyte_cdk/utils/mapping_helpers.py +3 -1
- airbyte_cdk/utils/message_utils.py +20 -11
- airbyte_cdk/utils/print_buffer.py +75 -0
- airbyte_cdk/utils/schema_inferrer.py +198 -28
- airbyte_cdk/utils/slice_hasher.py +30 -0
- airbyte_cdk/utils/spec_schema_transformations.py +6 -3
- airbyte_cdk/utils/stream_status_utils.py +8 -1
- airbyte_cdk/utils/traced_exception.py +61 -21
- airbyte_cdk-6.17.1.dev0.dist-info/METADATA +109 -0
- airbyte_cdk-6.17.1.dev0.dist-info/RECORD +350 -0
- {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/WHEEL +1 -2
- airbyte_cdk-6.17.1.dev0.dist-info/entry_points.txt +3 -0
- airbyte_cdk/sources/declarative/create_partial.py +0 -92
- airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
- airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
- airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
- airbyte_cdk/sources/deprecated/base_source.py +0 -94
- airbyte_cdk/sources/deprecated/client.py +0 -99
- airbyte_cdk/sources/singer/__init__.py +0 -8
- airbyte_cdk/sources/singer/singer_helpers.py +0 -304
- airbyte_cdk/sources/singer/source.py +0 -186
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
- airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
- airbyte_cdk/sources/streams/http/auth/core.py +0 -29
- airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
- airbyte_cdk/sources/streams/http/auth/token.py +0 -47
- airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
- airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
- airbyte_cdk/sources/utils/schema_models.py +0 -84
- airbyte_cdk-0.72.0.dist-info/METADATA +0 -243
- airbyte_cdk-0.72.0.dist-info/RECORD +0 -466
- airbyte_cdk-0.72.0.dist-info/top_level.txt +0 -3
- source_declarative_manifest/main.py +0 -29
- unit_tests/connector_builder/__init__.py +0 -3
- unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
- unit_tests/connector_builder/test_message_grouper.py +0 -713
- unit_tests/connector_builder/utils.py +0 -27
- unit_tests/destinations/test_destination.py +0 -243
- unit_tests/singer/test_singer_helpers.py +0 -56
- unit_tests/singer/test_singer_source.py +0 -112
- unit_tests/sources/__init__.py +0 -0
- unit_tests/sources/concurrent_source/__init__.py +0 -3
- unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
- unit_tests/sources/declarative/__init__.py +0 -3
- unit_tests/sources/declarative/auth/__init__.py +0 -3
- unit_tests/sources/declarative/auth/test_oauth.py +0 -331
- unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
- unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
- unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
- unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
- unit_tests/sources/declarative/checks/__init__.py +0 -3
- unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
- unit_tests/sources/declarative/decoders/__init__.py +0 -0
- unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
- unit_tests/sources/declarative/external_component.py +0 -13
- unit_tests/sources/declarative/extractors/__init__.py +0 -3
- unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
- unit_tests/sources/declarative/incremental/__init__.py +0 -0
- unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
- unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
- unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
- unit_tests/sources/declarative/interpolation/__init__.py +0 -3
- unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
- unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
- unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
- unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
- unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
- unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
- unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
- unit_tests/sources/declarative/parsers/__init__.py +0 -3
- unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
- unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
- unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1841
- unit_tests/sources/declarative/parsers/testing_components.py +0 -36
- unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
- unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
- unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
- unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
- unit_tests/sources/declarative/requesters/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
- unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
- unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
- unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
- unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
- unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
- unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
- unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
- unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
- unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
- unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
- unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
- unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
- unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
- unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
- unit_tests/sources/declarative/retrievers/__init__.py +0 -3
- unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
- unit_tests/sources/declarative/schema/__init__.py +0 -6
- unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
- unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
- unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
- unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
- unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
- unit_tests/sources/declarative/states/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
- unit_tests/sources/declarative/test_create_partial.py +0 -83
- unit_tests/sources/declarative/test_declarative_stream.py +0 -103
- unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
- unit_tests/sources/declarative/test_types.py +0 -39
- unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
- unit_tests/sources/file_based/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
- unit_tests/sources/file_based/config/__init__.py +0 -0
- unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
- unit_tests/sources/file_based/config/test_csv_format.py +0 -34
- unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
- unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
- unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
- unit_tests/sources/file_based/file_types/__init__.py +0 -0
- unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
- unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
- unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
- unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
- unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
- unit_tests/sources/file_based/helpers.py +0 -70
- unit_tests/sources/file_based/in_memory_files_source.py +0 -211
- unit_tests/sources/file_based/scenarios/__init__.py +0 -0
- unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
- unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
- unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
- unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
- unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
- unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
- unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
- unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
- unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
- unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
- unit_tests/sources/file_based/stream/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
- unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
- unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
- unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
- unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
- unit_tests/sources/file_based/test_scenarios.py +0 -253
- unit_tests/sources/file_based/test_schema_helpers.py +0 -346
- unit_tests/sources/fixtures/__init__.py +0 -3
- unit_tests/sources/fixtures/source_test_fixture.py +0 -153
- unit_tests/sources/message/__init__.py +0 -0
- unit_tests/sources/message/test_repository.py +0 -153
- unit_tests/sources/streams/__init__.py +0 -0
- unit_tests/sources/streams/concurrent/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
- unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
- unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
- unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
- unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
- unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
- unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
- unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
- unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
- unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
- unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
- unit_tests/sources/streams/http/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/test_auth.py +0 -173
- unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
- unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
- unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
- unit_tests/sources/streams/http/test_http.py +0 -635
- unit_tests/sources/streams/test_availability_strategy.py +0 -70
- unit_tests/sources/streams/test_call_rate.py +0 -300
- unit_tests/sources/streams/test_stream_read.py +0 -405
- unit_tests/sources/streams/test_streams_core.py +0 -184
- unit_tests/sources/test_abstract_source.py +0 -1442
- unit_tests/sources/test_concurrent_source.py +0 -112
- unit_tests/sources/test_config.py +0 -92
- unit_tests/sources/test_connector_state_manager.py +0 -482
- unit_tests/sources/test_http_logger.py +0 -252
- unit_tests/sources/test_integration_source.py +0 -86
- unit_tests/sources/test_source.py +0 -684
- unit_tests/sources/test_source_read.py +0 -460
- unit_tests/test/__init__.py +0 -0
- unit_tests/test/mock_http/__init__.py +0 -0
- unit_tests/test/mock_http/test_matcher.py +0 -53
- unit_tests/test/mock_http/test_mocker.py +0 -214
- unit_tests/test/mock_http/test_request.py +0 -117
- unit_tests/test/mock_http/test_response_builder.py +0 -177
- unit_tests/test/test_entrypoint_wrapper.py +0 -240
- unit_tests/utils/__init__.py +0 -0
- unit_tests/utils/test_datetime_format_inferrer.py +0 -60
- unit_tests/utils/test_mapping_helpers.py +0 -54
- unit_tests/utils/test_message_utils.py +0 -91
- unit_tests/utils/test_rate_limiting.py +0 -26
- unit_tests/utils/test_schema_inferrer.py +0 -202
- unit_tests/utils/test_secret_utils.py +0 -135
- unit_tests/utils/test_stream_status_utils.py +0 -61
- unit_tests/utils/test_traced_exception.py +0 -107
- /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
- {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
- {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
- {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
- {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/LICENSE.txt +0 -0
@@ -1,197 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
import unittest
|
5
|
-
from unittest.mock import Mock
|
6
|
-
|
7
|
-
from airbyte_cdk.models import AirbyteStream, SyncMode
|
8
|
-
from airbyte_cdk.sources.message import InMemoryMessageRepository
|
9
|
-
from airbyte_cdk.sources.streams.concurrent.availability_strategy import STREAM_AVAILABLE
|
10
|
-
from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, FinalStateCursor
|
11
|
-
from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
|
12
|
-
|
13
|
-
|
14
|
-
class ThreadBasedConcurrentStreamTest(unittest.TestCase):
|
15
|
-
def setUp(self):
|
16
|
-
self._partition_generator = Mock()
|
17
|
-
self._name = "name"
|
18
|
-
self._json_schema = {}
|
19
|
-
self._availability_strategy = Mock()
|
20
|
-
self._primary_key = []
|
21
|
-
self._cursor_field = None
|
22
|
-
self._logger = Mock()
|
23
|
-
self._cursor = Mock(spec=Cursor)
|
24
|
-
self._message_repository = InMemoryMessageRepository()
|
25
|
-
self._stream = DefaultStream(
|
26
|
-
self._partition_generator,
|
27
|
-
self._name,
|
28
|
-
self._json_schema,
|
29
|
-
self._availability_strategy,
|
30
|
-
self._primary_key,
|
31
|
-
self._cursor_field,
|
32
|
-
self._logger,
|
33
|
-
FinalStateCursor(stream_name=self._name, stream_namespace=None, message_repository=self._message_repository),
|
34
|
-
)
|
35
|
-
|
36
|
-
def test_get_json_schema(self):
|
37
|
-
json_schema = self._stream.get_json_schema()
|
38
|
-
assert json_schema == self._json_schema
|
39
|
-
|
40
|
-
def test_check_availability(self):
|
41
|
-
self._availability_strategy.check_availability.return_value = STREAM_AVAILABLE
|
42
|
-
availability = self._stream.check_availability()
|
43
|
-
assert availability == STREAM_AVAILABLE
|
44
|
-
self._availability_strategy.check_availability.assert_called_once_with(self._logger)
|
45
|
-
|
46
|
-
def test_check_for_error_raises_an_exception_if_any_of_the_futures_are_not_done(self):
|
47
|
-
futures = [Mock() for _ in range(3)]
|
48
|
-
for f in futures:
|
49
|
-
f.exception.return_value = None
|
50
|
-
futures[0].done.return_value = False
|
51
|
-
|
52
|
-
with self.assertRaises(Exception):
|
53
|
-
self._stream._check_for_errors(futures)
|
54
|
-
|
55
|
-
def test_check_for_error_raises_an_exception_if_any_of_the_futures_raised_an_exception(self):
|
56
|
-
futures = [Mock() for _ in range(3)]
|
57
|
-
for f in futures:
|
58
|
-
f.exception.return_value = None
|
59
|
-
futures[0].exception.return_value = Exception("error")
|
60
|
-
|
61
|
-
with self.assertRaises(Exception):
|
62
|
-
self._stream._check_for_errors(futures)
|
63
|
-
|
64
|
-
def test_as_airbyte_stream(self):
|
65
|
-
expected_airbyte_stream = AirbyteStream(
|
66
|
-
name=self._name,
|
67
|
-
json_schema=self._json_schema,
|
68
|
-
supported_sync_modes=[SyncMode.full_refresh],
|
69
|
-
source_defined_cursor=None,
|
70
|
-
default_cursor_field=None,
|
71
|
-
source_defined_primary_key=None,
|
72
|
-
namespace=None,
|
73
|
-
)
|
74
|
-
actual_airbyte_stream = self._stream.as_airbyte_stream()
|
75
|
-
|
76
|
-
assert expected_airbyte_stream == actual_airbyte_stream
|
77
|
-
|
78
|
-
def test_as_airbyte_stream_with_primary_key(self):
|
79
|
-
json_schema = {
|
80
|
-
"type": "object",
|
81
|
-
"properties": {
|
82
|
-
"id_a": {"type": ["null", "string"]},
|
83
|
-
"id_b": {"type": ["null", "string"]},
|
84
|
-
},
|
85
|
-
}
|
86
|
-
stream = DefaultStream(
|
87
|
-
self._partition_generator,
|
88
|
-
self._name,
|
89
|
-
json_schema,
|
90
|
-
self._availability_strategy,
|
91
|
-
["id"],
|
92
|
-
self._cursor_field,
|
93
|
-
self._logger,
|
94
|
-
FinalStateCursor(stream_name=self._name, stream_namespace=None, message_repository=self._message_repository),
|
95
|
-
)
|
96
|
-
|
97
|
-
expected_airbyte_stream = AirbyteStream(
|
98
|
-
name=self._name,
|
99
|
-
json_schema=json_schema,
|
100
|
-
supported_sync_modes=[SyncMode.full_refresh],
|
101
|
-
source_defined_cursor=None,
|
102
|
-
default_cursor_field=None,
|
103
|
-
source_defined_primary_key=[["id"]],
|
104
|
-
namespace=None,
|
105
|
-
)
|
106
|
-
|
107
|
-
airbyte_stream = stream.as_airbyte_stream()
|
108
|
-
assert expected_airbyte_stream == airbyte_stream
|
109
|
-
|
110
|
-
def test_as_airbyte_stream_with_composite_primary_key(self):
|
111
|
-
json_schema = {
|
112
|
-
"type": "object",
|
113
|
-
"properties": {
|
114
|
-
"id_a": {"type": ["null", "string"]},
|
115
|
-
"id_b": {"type": ["null", "string"]},
|
116
|
-
},
|
117
|
-
}
|
118
|
-
stream = DefaultStream(
|
119
|
-
self._partition_generator,
|
120
|
-
self._name,
|
121
|
-
json_schema,
|
122
|
-
self._availability_strategy,
|
123
|
-
["id_a", "id_b"],
|
124
|
-
self._cursor_field,
|
125
|
-
self._logger,
|
126
|
-
FinalStateCursor(stream_name=self._name, stream_namespace=None, message_repository=self._message_repository),
|
127
|
-
)
|
128
|
-
|
129
|
-
expected_airbyte_stream = AirbyteStream(
|
130
|
-
name=self._name,
|
131
|
-
json_schema=json_schema,
|
132
|
-
supported_sync_modes=[SyncMode.full_refresh],
|
133
|
-
source_defined_cursor=None,
|
134
|
-
default_cursor_field=None,
|
135
|
-
source_defined_primary_key=[["id_a", "id_b"]],
|
136
|
-
namespace=None,
|
137
|
-
)
|
138
|
-
|
139
|
-
airbyte_stream = stream.as_airbyte_stream()
|
140
|
-
assert expected_airbyte_stream == airbyte_stream
|
141
|
-
|
142
|
-
def test_as_airbyte_stream_with_a_cursor(self):
|
143
|
-
json_schema = {
|
144
|
-
"type": "object",
|
145
|
-
"properties": {
|
146
|
-
"id": {"type": ["null", "string"]},
|
147
|
-
"date": {"type": ["null", "string"]},
|
148
|
-
},
|
149
|
-
}
|
150
|
-
stream = DefaultStream(
|
151
|
-
self._partition_generator,
|
152
|
-
self._name,
|
153
|
-
json_schema,
|
154
|
-
self._availability_strategy,
|
155
|
-
self._primary_key,
|
156
|
-
"date",
|
157
|
-
self._logger,
|
158
|
-
FinalStateCursor(stream_name=self._name, stream_namespace=None, message_repository=self._message_repository),
|
159
|
-
)
|
160
|
-
|
161
|
-
expected_airbyte_stream = AirbyteStream(
|
162
|
-
name=self._name,
|
163
|
-
json_schema=json_schema,
|
164
|
-
supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental],
|
165
|
-
source_defined_cursor=True,
|
166
|
-
default_cursor_field=["date"],
|
167
|
-
source_defined_primary_key=None,
|
168
|
-
namespace=None,
|
169
|
-
)
|
170
|
-
|
171
|
-
airbyte_stream = stream.as_airbyte_stream()
|
172
|
-
assert expected_airbyte_stream == airbyte_stream
|
173
|
-
|
174
|
-
def test_as_airbyte_stream_with_namespace(self):
|
175
|
-
stream = DefaultStream(
|
176
|
-
self._partition_generator,
|
177
|
-
self._name,
|
178
|
-
self._json_schema,
|
179
|
-
self._availability_strategy,
|
180
|
-
self._primary_key,
|
181
|
-
self._cursor_field,
|
182
|
-
self._logger,
|
183
|
-
FinalStateCursor(stream_name=self._name, stream_namespace=None, message_repository=self._message_repository),
|
184
|
-
namespace="test",
|
185
|
-
)
|
186
|
-
expected_airbyte_stream = AirbyteStream(
|
187
|
-
name=self._name,
|
188
|
-
json_schema=self._json_schema,
|
189
|
-
supported_sync_modes=[SyncMode.full_refresh],
|
190
|
-
source_defined_cursor=None,
|
191
|
-
default_cursor_field=None,
|
192
|
-
source_defined_primary_key=None,
|
193
|
-
namespace="test",
|
194
|
-
)
|
195
|
-
actual_airbyte_stream = stream.as_airbyte_stream()
|
196
|
-
|
197
|
-
assert expected_airbyte_stream == actual_airbyte_stream
|
@@ -1,90 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
import unittest
|
5
|
-
from queue import Queue
|
6
|
-
from typing import Callable, Iterable, List
|
7
|
-
from unittest.mock import Mock, patch
|
8
|
-
|
9
|
-
from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel
|
10
|
-
from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager
|
11
|
-
from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
|
12
|
-
from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer
|
13
|
-
from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
|
14
|
-
from airbyte_cdk.sources.streams.concurrent.partitions.types import QueueItem
|
15
|
-
|
16
|
-
_SOME_PARTITIONS: List[Partition] = [Mock(spec=Partition), Mock(spec=Partition)]
|
17
|
-
|
18
|
-
|
19
|
-
class PartitionEnqueuerTest(unittest.TestCase):
|
20
|
-
def setUp(self) -> None:
|
21
|
-
self._queue: Queue[QueueItem] = Queue()
|
22
|
-
self._thread_pool_manager = Mock(spec=ThreadPoolManager)
|
23
|
-
self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = False
|
24
|
-
self._partition_generator = PartitionEnqueuer(self._queue, self._thread_pool_manager)
|
25
|
-
|
26
|
-
@patch("airbyte_cdk.sources.streams.concurrent.partition_enqueuer.time.sleep")
|
27
|
-
def test_given_no_partitions_when_generate_partitions_then_do_not_wait(self, mocked_sleep):
|
28
|
-
self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = True # shouldn't be called but just in case
|
29
|
-
stream = self._a_stream([])
|
30
|
-
|
31
|
-
self._partition_generator.generate_partitions(stream)
|
32
|
-
|
33
|
-
assert mocked_sleep.call_count == 0
|
34
|
-
|
35
|
-
def test_given_no_partitions_when_generate_partitions_then_only_push_sentinel(self):
|
36
|
-
self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = True
|
37
|
-
stream = self._a_stream([])
|
38
|
-
|
39
|
-
self._partition_generator.generate_partitions(stream)
|
40
|
-
|
41
|
-
assert self._consume_queue() == [PartitionGenerationCompletedSentinel(stream)]
|
42
|
-
|
43
|
-
def test_given_partitions_when_generate_partitions_then_return_partitions_before_sentinel(self):
|
44
|
-
self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.return_value = False
|
45
|
-
stream = self._a_stream(_SOME_PARTITIONS)
|
46
|
-
|
47
|
-
self._partition_generator.generate_partitions(stream)
|
48
|
-
|
49
|
-
assert self._consume_queue() == _SOME_PARTITIONS + [PartitionGenerationCompletedSentinel(stream)]
|
50
|
-
|
51
|
-
@patch("airbyte_cdk.sources.streams.concurrent.partition_enqueuer.time.sleep")
|
52
|
-
def test_given_partition_but_limit_reached_when_generate_partitions_then_wait_until_not_hitting_limit(self, mocked_sleep):
|
53
|
-
self._thread_pool_manager.prune_to_validate_has_reached_futures_limit.side_effect = [True, True, False]
|
54
|
-
stream = self._a_stream([Mock(spec=Partition)])
|
55
|
-
|
56
|
-
self._partition_generator.generate_partitions(stream)
|
57
|
-
|
58
|
-
assert mocked_sleep.call_count == 2
|
59
|
-
|
60
|
-
def test_given_exception_when_generate_partitions_then_raise(self):
|
61
|
-
stream = Mock(spec=AbstractStream)
|
62
|
-
exception = ValueError()
|
63
|
-
stream.generate_partitions.side_effect = self._partitions_before_raising(_SOME_PARTITIONS, exception)
|
64
|
-
|
65
|
-
self._partition_generator.generate_partitions(stream)
|
66
|
-
|
67
|
-
assert self._consume_queue() == _SOME_PARTITIONS + [exception]
|
68
|
-
|
69
|
-
def _partitions_before_raising(self, partitions: List[Partition], exception: Exception) -> Callable[[], Iterable[Partition]]:
|
70
|
-
def inner_function() -> Iterable[Partition]:
|
71
|
-
for partition in partitions:
|
72
|
-
yield partition
|
73
|
-
raise exception
|
74
|
-
|
75
|
-
return inner_function
|
76
|
-
|
77
|
-
@staticmethod
|
78
|
-
def _a_stream(partitions: List[Partition]) -> AbstractStream:
|
79
|
-
stream = Mock(spec=AbstractStream)
|
80
|
-
stream.generate_partitions.return_value = iter(partitions)
|
81
|
-
return stream
|
82
|
-
|
83
|
-
def _consume_queue(self) -> List[QueueItem]:
|
84
|
-
queue_content: List[QueueItem] = []
|
85
|
-
while queue_item := self._queue.get():
|
86
|
-
if isinstance(queue_item, (PartitionGenerationCompletedSentinel, Exception)):
|
87
|
-
queue_content.append(queue_item)
|
88
|
-
break
|
89
|
-
queue_content.append(queue_item)
|
90
|
-
return queue_content
|
@@ -1,67 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
import unittest
|
5
|
-
from queue import Queue
|
6
|
-
from typing import Callable, Iterable, List
|
7
|
-
from unittest.mock import Mock
|
8
|
-
|
9
|
-
import pytest
|
10
|
-
from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader
|
11
|
-
from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
|
12
|
-
from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
|
13
|
-
from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel, QueueItem
|
14
|
-
|
15
|
-
_RECORDS = [
|
16
|
-
Record({"id": 1, "name": "Jack"}, "stream"),
|
17
|
-
Record({"id": 2, "name": "John"}, "stream"),
|
18
|
-
]
|
19
|
-
|
20
|
-
|
21
|
-
class PartitionReaderTest(unittest.TestCase):
|
22
|
-
def setUp(self) -> None:
|
23
|
-
self._queue: Queue[QueueItem] = Queue()
|
24
|
-
self._partition_reader = PartitionReader(self._queue)
|
25
|
-
|
26
|
-
def test_given_no_records_when_process_partition_then_only_emit_sentinel(self):
|
27
|
-
self._partition_reader.process_partition(self._a_partition([]))
|
28
|
-
|
29
|
-
while queue_item := self._queue.get():
|
30
|
-
if not isinstance(queue_item, PartitionCompleteSentinel):
|
31
|
-
pytest.fail("Only one PartitionCompleteSentinel is expected")
|
32
|
-
break
|
33
|
-
|
34
|
-
def test_given_read_partition_successful_when_process_partition_then_queue_records_and_sentinel(self):
|
35
|
-
self._partition_reader.process_partition(self._a_partition(_RECORDS))
|
36
|
-
|
37
|
-
actual_records = []
|
38
|
-
while queue_item := self._queue.get():
|
39
|
-
if isinstance(queue_item, PartitionCompleteSentinel):
|
40
|
-
break
|
41
|
-
actual_records.append(queue_item)
|
42
|
-
|
43
|
-
assert _RECORDS == actual_records
|
44
|
-
|
45
|
-
def test_given_exception_when_process_partition_then_queue_records_and_raise_exception(self):
|
46
|
-
partition = Mock()
|
47
|
-
exception = ValueError()
|
48
|
-
partition.read.side_effect = self._read_with_exception(_RECORDS, exception)
|
49
|
-
|
50
|
-
self._partition_reader.process_partition(partition)
|
51
|
-
|
52
|
-
for i in range(len(_RECORDS)):
|
53
|
-
assert self._queue.get() == _RECORDS[i]
|
54
|
-
assert self._queue.get() == exception
|
55
|
-
|
56
|
-
def _a_partition(self, records: List[Record]) -> Partition:
|
57
|
-
partition = Mock(spec=Partition)
|
58
|
-
partition.read.return_value = iter(records)
|
59
|
-
return partition
|
60
|
-
|
61
|
-
@staticmethod
|
62
|
-
def _read_with_exception(records: List[Record], exception: Exception) -> Callable[[], Iterable[Record]]:
|
63
|
-
def mocked_function() -> Iterable[Record]:
|
64
|
-
yield from records
|
65
|
-
raise exception
|
66
|
-
|
67
|
-
return mocked_function
|
@@ -1,106 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
from concurrent.futures import Future, ThreadPoolExecutor
|
5
|
-
from unittest import TestCase
|
6
|
-
from unittest.mock import Mock
|
7
|
-
|
8
|
-
from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager
|
9
|
-
|
10
|
-
|
11
|
-
class ThreadPoolManagerTest(TestCase):
|
12
|
-
def setUp(self):
|
13
|
-
self._threadpool = Mock(spec=ThreadPoolExecutor)
|
14
|
-
self._thread_pool_manager = ThreadPoolManager(self._threadpool, Mock(), max_concurrent_tasks=1)
|
15
|
-
self._fn = lambda x: x
|
16
|
-
self._arg = "arg"
|
17
|
-
|
18
|
-
def test_submit_calls_underlying_thread_pool(self):
|
19
|
-
self._thread_pool_manager.submit(self._fn, self._arg)
|
20
|
-
self._threadpool.submit.assert_called_with(self._fn, self._arg)
|
21
|
-
|
22
|
-
assert len(self._thread_pool_manager._futures) == 1
|
23
|
-
|
24
|
-
def test_given_no_exceptions_when_shutdown_if_exception_then_do_not_raise(self):
|
25
|
-
future = Mock(spec=Future)
|
26
|
-
future.exception.return_value = None
|
27
|
-
future.done.side_effect = [True, True]
|
28
|
-
|
29
|
-
self._thread_pool_manager._futures = [future]
|
30
|
-
self._thread_pool_manager.prune_to_validate_has_reached_futures_limit()
|
31
|
-
|
32
|
-
self._thread_pool_manager.shutdown_if_exception() # do not raise
|
33
|
-
|
34
|
-
def test_given_exception_when_shutdown_if_exception_then_raise(self):
|
35
|
-
future = Mock(spec=Future)
|
36
|
-
future.exception.return_value = RuntimeError
|
37
|
-
future.done.side_effect = [True, True]
|
38
|
-
|
39
|
-
self._thread_pool_manager._futures = [future]
|
40
|
-
self._thread_pool_manager.prune_to_validate_has_reached_futures_limit()
|
41
|
-
|
42
|
-
with self.assertRaises(RuntimeError):
|
43
|
-
self._thread_pool_manager.shutdown_if_exception()
|
44
|
-
|
45
|
-
def test_given_exception_during_pruning_when_check_for_errors_and_shutdown_then_shutdown_and_raise(self):
|
46
|
-
future = Mock(spec=Future)
|
47
|
-
future.exception.return_value = RuntimeError
|
48
|
-
future.done.side_effect = [True, True]
|
49
|
-
|
50
|
-
self._thread_pool_manager._futures = [future]
|
51
|
-
self._thread_pool_manager.prune_to_validate_has_reached_futures_limit()
|
52
|
-
|
53
|
-
with self.assertRaises(RuntimeError):
|
54
|
-
self._thread_pool_manager.check_for_errors_and_shutdown()
|
55
|
-
self._threadpool.shutdown.assert_called_with(wait=False, cancel_futures=True)
|
56
|
-
|
57
|
-
def test_shutdown(self):
|
58
|
-
self._thread_pool_manager.shutdown()
|
59
|
-
self._threadpool.shutdown.assert_called_with(wait=False, cancel_futures=True)
|
60
|
-
|
61
|
-
def test_is_done_is_false_if_not_all_futures_are_done(self):
|
62
|
-
future = Mock(spec=Future)
|
63
|
-
future.done.return_value = False
|
64
|
-
|
65
|
-
self._thread_pool_manager._futures = [future]
|
66
|
-
|
67
|
-
assert not self._thread_pool_manager.is_done()
|
68
|
-
|
69
|
-
def test_is_done_is_true_if_all_futures_are_done(self):
|
70
|
-
future = Mock(spec=Future)
|
71
|
-
future.done.return_value = True
|
72
|
-
|
73
|
-
self._thread_pool_manager._futures = [future]
|
74
|
-
|
75
|
-
assert self._thread_pool_manager.is_done()
|
76
|
-
|
77
|
-
def test_threadpool_shutdown_if_errors(self):
|
78
|
-
future = Mock(spec=Future)
|
79
|
-
future.exception.return_value = RuntimeError
|
80
|
-
|
81
|
-
self._thread_pool_manager._futures = [future]
|
82
|
-
|
83
|
-
with self.assertRaises(RuntimeError):
|
84
|
-
self._thread_pool_manager.check_for_errors_and_shutdown()
|
85
|
-
self._threadpool.shutdown.assert_called_with(wait=False, cancel_futures=True)
|
86
|
-
|
87
|
-
def test_check_for_errors_and_shutdown_raises_error_if_futures_are_not_done(self):
|
88
|
-
future = Mock(spec=Future)
|
89
|
-
future.exception.return_value = None
|
90
|
-
future.done.return_value = False
|
91
|
-
|
92
|
-
self._thread_pool_manager._futures = [future]
|
93
|
-
|
94
|
-
with self.assertRaises(RuntimeError):
|
95
|
-
self._thread_pool_manager.check_for_errors_and_shutdown()
|
96
|
-
self._threadpool.shutdown.assert_called_with(wait=False, cancel_futures=True)
|
97
|
-
|
98
|
-
def test_check_for_errors_and_shutdown_does_not_raise_error_if_futures_are_done(self):
|
99
|
-
future = Mock(spec=Future)
|
100
|
-
future.exception.return_value = None
|
101
|
-
future.done.return_value = True
|
102
|
-
|
103
|
-
self._thread_pool_manager._futures = [future]
|
104
|
-
|
105
|
-
self._thread_pool_manager.check_for_errors_and_shutdown()
|
106
|
-
self._threadpool.shutdown.assert_called_with(wait=False, cancel_futures=True)
|
File without changes
|
File without changes
|
@@ -1,173 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
|
6
|
-
import logging
|
7
|
-
|
8
|
-
import pytest
|
9
|
-
from airbyte_cdk.sources.streams.http.auth import (
|
10
|
-
BasicHttpAuthenticator,
|
11
|
-
MultipleTokenAuthenticator,
|
12
|
-
NoAuth,
|
13
|
-
Oauth2Authenticator,
|
14
|
-
TokenAuthenticator,
|
15
|
-
)
|
16
|
-
|
17
|
-
LOGGER = logging.getLogger(__name__)
|
18
|
-
|
19
|
-
|
20
|
-
def test_token_authenticator():
|
21
|
-
"""
|
22
|
-
Should match passed in token, no matter how many times token is retrieved.
|
23
|
-
"""
|
24
|
-
token = TokenAuthenticator("test-token")
|
25
|
-
header = token.get_auth_header()
|
26
|
-
assert {"Authorization": "Bearer test-token"} == header
|
27
|
-
header = token.get_auth_header()
|
28
|
-
assert {"Authorization": "Bearer test-token"} == header
|
29
|
-
|
30
|
-
|
31
|
-
def test_multiple_token_authenticator():
|
32
|
-
token = MultipleTokenAuthenticator(["token1", "token2"])
|
33
|
-
header1 = token.get_auth_header()
|
34
|
-
assert {"Authorization": "Bearer token1"} == header1
|
35
|
-
header2 = token.get_auth_header()
|
36
|
-
assert {"Authorization": "Bearer token2"} == header2
|
37
|
-
header3 = token.get_auth_header()
|
38
|
-
assert {"Authorization": "Bearer token1"} == header3
|
39
|
-
|
40
|
-
|
41
|
-
def test_no_auth():
|
42
|
-
"""
|
43
|
-
Should always return empty body, no matter how many times token is retrieved.
|
44
|
-
"""
|
45
|
-
no_auth = NoAuth()
|
46
|
-
assert {} == no_auth.get_auth_header()
|
47
|
-
no_auth = NoAuth()
|
48
|
-
assert {} == no_auth.get_auth_header()
|
49
|
-
|
50
|
-
|
51
|
-
def test_basic_authenticator():
|
52
|
-
token = BasicHttpAuthenticator("client_id", "client_secret")
|
53
|
-
header = token.get_auth_header()
|
54
|
-
assert {"Authorization": "Basic Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ="} == header
|
55
|
-
|
56
|
-
|
57
|
-
class TestOauth2Authenticator:
|
58
|
-
"""
|
59
|
-
Test class for OAuth2Authenticator.
|
60
|
-
"""
|
61
|
-
|
62
|
-
refresh_endpoint = "https://some_url.com/v1"
|
63
|
-
client_id = "client_id"
|
64
|
-
client_secret = "client_secret"
|
65
|
-
refresh_token = "refresh_token"
|
66
|
-
refresh_access_token_headers = {"Header_1": "value 1", "Header_2": "value 2"}
|
67
|
-
refresh_access_token_authenticator = BasicHttpAuthenticator(client_id, client_secret)
|
68
|
-
|
69
|
-
def test_get_auth_header_fresh(self, mocker):
|
70
|
-
"""
|
71
|
-
Should not retrieve new token if current token is valid.
|
72
|
-
"""
|
73
|
-
oauth = Oauth2Authenticator(
|
74
|
-
TestOauth2Authenticator.refresh_endpoint,
|
75
|
-
TestOauth2Authenticator.client_id,
|
76
|
-
TestOauth2Authenticator.client_secret,
|
77
|
-
TestOauth2Authenticator.refresh_token,
|
78
|
-
)
|
79
|
-
|
80
|
-
mocker.patch.object(Oauth2Authenticator, "refresh_access_token", return_value=("access_token", 1000))
|
81
|
-
header = oauth.get_auth_header()
|
82
|
-
assert {"Authorization": "Bearer access_token"} == header
|
83
|
-
|
84
|
-
def test_get_auth_header_expired(self, mocker):
|
85
|
-
"""
|
86
|
-
Should retrieve new token if current token is expired.
|
87
|
-
"""
|
88
|
-
oauth = Oauth2Authenticator(
|
89
|
-
TestOauth2Authenticator.refresh_endpoint,
|
90
|
-
TestOauth2Authenticator.client_id,
|
91
|
-
TestOauth2Authenticator.client_secret,
|
92
|
-
TestOauth2Authenticator.refresh_token,
|
93
|
-
)
|
94
|
-
|
95
|
-
expire_immediately = 0
|
96
|
-
mocker.patch.object(Oauth2Authenticator, "refresh_access_token", return_value=("access_token_1", expire_immediately))
|
97
|
-
oauth.get_auth_header() # Set the first expired token.
|
98
|
-
|
99
|
-
valid_100_secs = 100
|
100
|
-
mocker.patch.object(Oauth2Authenticator, "refresh_access_token", return_value=("access_token_2", valid_100_secs))
|
101
|
-
header = oauth.get_auth_header()
|
102
|
-
assert {"Authorization": "Bearer access_token_2"} == header
|
103
|
-
|
104
|
-
def test_refresh_request_body(self):
|
105
|
-
"""
|
106
|
-
Request body should match given configuration.
|
107
|
-
"""
|
108
|
-
scopes = ["scope1", "scope2"]
|
109
|
-
oauth = Oauth2Authenticator(
|
110
|
-
TestOauth2Authenticator.refresh_endpoint,
|
111
|
-
TestOauth2Authenticator.client_id,
|
112
|
-
TestOauth2Authenticator.client_secret,
|
113
|
-
TestOauth2Authenticator.refresh_token,
|
114
|
-
scopes,
|
115
|
-
)
|
116
|
-
body = oauth.get_refresh_request_body()
|
117
|
-
expected = {
|
118
|
-
"grant_type": "refresh_token",
|
119
|
-
"client_id": "client_id",
|
120
|
-
"client_secret": "client_secret",
|
121
|
-
"refresh_token": "refresh_token",
|
122
|
-
"scopes": scopes,
|
123
|
-
}
|
124
|
-
assert body == expected
|
125
|
-
|
126
|
-
def test_refresh_access_token(self, requests_mock):
|
127
|
-
mock_refresh_token_call = requests_mock.post(
|
128
|
-
TestOauth2Authenticator.refresh_endpoint, json={"access_token": "token", "expires_in": 10}
|
129
|
-
)
|
130
|
-
|
131
|
-
oauth = Oauth2Authenticator(
|
132
|
-
TestOauth2Authenticator.refresh_endpoint,
|
133
|
-
TestOauth2Authenticator.client_id,
|
134
|
-
TestOauth2Authenticator.client_secret,
|
135
|
-
TestOauth2Authenticator.refresh_token,
|
136
|
-
refresh_access_token_headers=TestOauth2Authenticator.refresh_access_token_headers,
|
137
|
-
)
|
138
|
-
|
139
|
-
token, expires_in = oauth.refresh_access_token()
|
140
|
-
|
141
|
-
assert isinstance(expires_in, int)
|
142
|
-
assert ("token", 10) == (token, expires_in)
|
143
|
-
for header in self.refresh_access_token_headers:
|
144
|
-
assert header in mock_refresh_token_call.last_request.headers
|
145
|
-
assert self.refresh_access_token_headers[header] == mock_refresh_token_call.last_request.headers[header]
|
146
|
-
assert mock_refresh_token_call.called
|
147
|
-
|
148
|
-
@pytest.mark.parametrize("error_code", (429, 500, 502, 504))
|
149
|
-
def test_refresh_access_token_retry(self, error_code, requests_mock):
|
150
|
-
oauth = Oauth2Authenticator(
|
151
|
-
TestOauth2Authenticator.refresh_endpoint,
|
152
|
-
TestOauth2Authenticator.client_id,
|
153
|
-
TestOauth2Authenticator.client_secret,
|
154
|
-
TestOauth2Authenticator.refresh_token,
|
155
|
-
)
|
156
|
-
requests_mock.post(
|
157
|
-
TestOauth2Authenticator.refresh_endpoint,
|
158
|
-
[{"status_code": error_code}, {"status_code": error_code}, {"json": {"access_token": "token", "expires_in": 10}}],
|
159
|
-
)
|
160
|
-
token, expires_in = oauth.refresh_access_token()
|
161
|
-
assert (token, expires_in) == ("token", 10)
|
162
|
-
assert requests_mock.call_count == 3
|
163
|
-
|
164
|
-
def test_refresh_access_authenticator(self):
|
165
|
-
oauth = Oauth2Authenticator(
|
166
|
-
TestOauth2Authenticator.refresh_endpoint,
|
167
|
-
TestOauth2Authenticator.client_id,
|
168
|
-
TestOauth2Authenticator.client_secret,
|
169
|
-
TestOauth2Authenticator.refresh_token,
|
170
|
-
refresh_access_token_authenticator=TestOauth2Authenticator.refresh_access_token_authenticator,
|
171
|
-
)
|
172
|
-
expected_headers = {"Authorization": "Basic Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ="}
|
173
|
-
assert expected_headers == oauth.get_refresh_access_token_headers()
|
File without changes
|