airbyte-cdk 0.72.1__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- airbyte_cdk/__init__.py +355 -6
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +29 -10
- airbyte_cdk/connector.py +24 -24
- airbyte_cdk/connector_builder/README.md +53 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
- airbyte_cdk/connector_builder/main.py +45 -13
- airbyte_cdk/connector_builder/message_grouper.py +189 -50
- airbyte_cdk/connector_builder/models.py +3 -2
- airbyte_cdk/destinations/__init__.py +4 -3
- airbyte_cdk/destinations/destination.py +54 -20
- airbyte_cdk/destinations/vector_db_based/README.md +37 -0
- airbyte_cdk/destinations/vector_db_based/config.py +40 -17
- airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
- airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
- airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
- airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
- airbyte_cdk/entrypoint.py +153 -44
- airbyte_cdk/exception_handler.py +21 -3
- airbyte_cdk/logger.py +30 -44
- airbyte_cdk/models/__init__.py +13 -2
- airbyte_cdk/models/airbyte_protocol.py +86 -1
- airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
- airbyte_cdk/models/file_transfer_record_message.py +13 -0
- airbyte_cdk/models/well_known_types.py +1 -1
- airbyte_cdk/sources/__init__.py +5 -1
- airbyte_cdk/sources/abstract_source.py +125 -79
- airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
- airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
- airbyte_cdk/sources/config.py +3 -2
- airbyte_cdk/sources/connector_state_manager.py +49 -83
- airbyte_cdk/sources/declarative/async_job/job.py +52 -0
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
- airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
- airbyte_cdk/sources/declarative/async_job/status.py +24 -0
- airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
- airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
- airbyte_cdk/sources/declarative/auth/token.py +28 -10
- airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
- airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
- airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
- airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +490 -0
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1185 -85
- airbyte_cdk/sources/declarative/declarative_source.py +5 -2
- airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
- airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
- airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
- airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
- airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
- airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
- airbyte_cdk/sources/declarative/extractors/record_filter.py +63 -8
- airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
- airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
- airbyte_cdk/sources/declarative/incremental/__init__.py +31 -3
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +346 -0
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
- airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +173 -74
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
- airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
- airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
- airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
- airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
- airbyte_cdk/sources/declarative/models/__init__.py +1 -1
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1319 -603
- airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1759 -225
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
- airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
- airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
- airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
- airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
- airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
- airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
- airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
- airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
- airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
- airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
- airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +229 -73
- airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
- airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
- airbyte_cdk/sources/declarative/spec/spec.py +12 -5
- airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
- airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
- airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
- airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
- airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
- airbyte_cdk/sources/declarative/types.py +19 -110
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
- airbyte_cdk/sources/embedded/base_integration.py +16 -5
- airbyte_cdk/sources/embedded/catalog.py +16 -4
- airbyte_cdk/sources/embedded/runner.py +19 -3
- airbyte_cdk/sources/embedded/tools.py +5 -2
- airbyte_cdk/sources/file_based/README.md +152 -0
- airbyte_cdk/sources/file_based/__init__.py +24 -0
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +47 -10
- airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
- airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
- airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
- airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
- airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
- airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
- airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
- airbyte_cdk/sources/file_based/exceptions.py +18 -15
- airbyte_cdk/sources/file_based/file_based_source.py +140 -33
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +69 -5
- airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +141 -41
- airbyte_cdk/sources/file_based/remote_file.py +1 -1
- airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
- airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +147 -45
- airbyte_cdk/sources/http_logger.py +8 -3
- airbyte_cdk/sources/message/__init__.py +7 -1
- airbyte_cdk/sources/message/repository.py +18 -4
- airbyte_cdk/sources/source.py +42 -38
- airbyte_cdk/sources/streams/__init__.py +2 -2
- airbyte_cdk/sources/streams/availability_strategy.py +54 -3
- airbyte_cdk/sources/streams/call_rate.py +64 -21
- airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
- airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
- airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
- airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
- airbyte_cdk/sources/streams/concurrent/README.md +7 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
- airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
- airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
- airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
- airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
- airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
- airbyte_cdk/sources/streams/core.py +412 -87
- airbyte_cdk/sources/streams/http/__init__.py +2 -1
- airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
- airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
- airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
- airbyte_cdk/sources/streams/http/exceptions.py +27 -7
- airbyte_cdk/sources/streams/http/http.py +369 -246
- airbyte_cdk/sources/streams/http/http_client.py +531 -0
- airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
- airbyte_cdk/sources/types.py +154 -0
- airbyte_cdk/sources/utils/record_helper.py +36 -21
- airbyte_cdk/sources/utils/schema_helpers.py +13 -6
- airbyte_cdk/sources/utils/slice_logger.py +4 -1
- airbyte_cdk/sources/utils/transform.py +54 -20
- airbyte_cdk/sql/_util/hashing.py +34 -0
- airbyte_cdk/sql/_util/name_normalizers.py +92 -0
- airbyte_cdk/sql/constants.py +32 -0
- airbyte_cdk/sql/exceptions.py +235 -0
- airbyte_cdk/sql/secrets.py +123 -0
- airbyte_cdk/sql/shared/__init__.py +15 -0
- airbyte_cdk/sql/shared/catalog_providers.py +145 -0
- airbyte_cdk/sql/shared/sql_processor.py +786 -0
- airbyte_cdk/sql/types.py +160 -0
- airbyte_cdk/test/catalog_builder.py +70 -18
- airbyte_cdk/test/entrypoint_wrapper.py +117 -42
- airbyte_cdk/test/mock_http/__init__.py +1 -1
- airbyte_cdk/test/mock_http/matcher.py +6 -0
- airbyte_cdk/test/mock_http/mocker.py +57 -10
- airbyte_cdk/test/mock_http/request.py +19 -3
- airbyte_cdk/test/mock_http/response.py +3 -1
- airbyte_cdk/test/mock_http/response_builder.py +32 -16
- airbyte_cdk/test/state_builder.py +18 -10
- airbyte_cdk/test/utils/__init__.py +1 -0
- airbyte_cdk/test/utils/data.py +24 -0
- airbyte_cdk/test/utils/http_mocking.py +16 -0
- airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
- airbyte_cdk/test/utils/reading.py +26 -0
- airbyte_cdk/utils/__init__.py +2 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
- airbyte_cdk/utils/analytics_message.py +10 -2
- airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
- airbyte_cdk/utils/event_timing.py +10 -10
- airbyte_cdk/utils/mapping_helpers.py +3 -1
- airbyte_cdk/utils/message_utils.py +20 -11
- airbyte_cdk/utils/print_buffer.py +75 -0
- airbyte_cdk/utils/schema_inferrer.py +198 -28
- airbyte_cdk/utils/slice_hasher.py +30 -0
- airbyte_cdk/utils/spec_schema_transformations.py +6 -3
- airbyte_cdk/utils/stream_status_utils.py +8 -1
- airbyte_cdk/utils/traced_exception.py +61 -21
- airbyte_cdk-6.17.1.dev0.dist-info/METADATA +109 -0
- airbyte_cdk-6.17.1.dev0.dist-info/RECORD +350 -0
- {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/WHEEL +1 -2
- airbyte_cdk-6.17.1.dev0.dist-info/entry_points.txt +3 -0
- airbyte_cdk/sources/declarative/create_partial.py +0 -92
- airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
- airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
- airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
- airbyte_cdk/sources/deprecated/base_source.py +0 -94
- airbyte_cdk/sources/deprecated/client.py +0 -99
- airbyte_cdk/sources/singer/__init__.py +0 -8
- airbyte_cdk/sources/singer/singer_helpers.py +0 -304
- airbyte_cdk/sources/singer/source.py +0 -186
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
- airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
- airbyte_cdk/sources/streams/http/auth/core.py +0 -29
- airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
- airbyte_cdk/sources/streams/http/auth/token.py +0 -47
- airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
- airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
- airbyte_cdk/sources/utils/schema_models.py +0 -84
- airbyte_cdk-0.72.1.dist-info/METADATA +0 -243
- airbyte_cdk-0.72.1.dist-info/RECORD +0 -466
- airbyte_cdk-0.72.1.dist-info/top_level.txt +0 -3
- source_declarative_manifest/main.py +0 -29
- unit_tests/connector_builder/__init__.py +0 -3
- unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
- unit_tests/connector_builder/test_message_grouper.py +0 -713
- unit_tests/connector_builder/utils.py +0 -27
- unit_tests/destinations/test_destination.py +0 -243
- unit_tests/singer/test_singer_helpers.py +0 -56
- unit_tests/singer/test_singer_source.py +0 -112
- unit_tests/sources/__init__.py +0 -0
- unit_tests/sources/concurrent_source/__init__.py +0 -3
- unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
- unit_tests/sources/declarative/__init__.py +0 -3
- unit_tests/sources/declarative/auth/__init__.py +0 -3
- unit_tests/sources/declarative/auth/test_oauth.py +0 -331
- unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
- unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
- unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
- unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
- unit_tests/sources/declarative/checks/__init__.py +0 -3
- unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
- unit_tests/sources/declarative/decoders/__init__.py +0 -0
- unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
- unit_tests/sources/declarative/external_component.py +0 -13
- unit_tests/sources/declarative/extractors/__init__.py +0 -3
- unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
- unit_tests/sources/declarative/incremental/__init__.py +0 -0
- unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
- unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
- unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
- unit_tests/sources/declarative/interpolation/__init__.py +0 -3
- unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
- unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
- unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
- unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
- unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
- unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
- unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
- unit_tests/sources/declarative/parsers/__init__.py +0 -3
- unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
- unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
- unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1847
- unit_tests/sources/declarative/parsers/testing_components.py +0 -36
- unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
- unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
- unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
- unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
- unit_tests/sources/declarative/requesters/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
- unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
- unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
- unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
- unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
- unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
- unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
- unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
- unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
- unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
- unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
- unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
- unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
- unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
- unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
- unit_tests/sources/declarative/retrievers/__init__.py +0 -3
- unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
- unit_tests/sources/declarative/schema/__init__.py +0 -6
- unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
- unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
- unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
- unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
- unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
- unit_tests/sources/declarative/states/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
- unit_tests/sources/declarative/test_create_partial.py +0 -83
- unit_tests/sources/declarative/test_declarative_stream.py +0 -103
- unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
- unit_tests/sources/declarative/test_types.py +0 -39
- unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
- unit_tests/sources/file_based/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
- unit_tests/sources/file_based/config/__init__.py +0 -0
- unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
- unit_tests/sources/file_based/config/test_csv_format.py +0 -34
- unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
- unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
- unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
- unit_tests/sources/file_based/file_types/__init__.py +0 -0
- unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
- unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
- unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
- unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
- unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
- unit_tests/sources/file_based/helpers.py +0 -70
- unit_tests/sources/file_based/in_memory_files_source.py +0 -211
- unit_tests/sources/file_based/scenarios/__init__.py +0 -0
- unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
- unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
- unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
- unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
- unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
- unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
- unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
- unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
- unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
- unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
- unit_tests/sources/file_based/stream/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
- unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
- unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
- unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
- unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
- unit_tests/sources/file_based/test_scenarios.py +0 -253
- unit_tests/sources/file_based/test_schema_helpers.py +0 -346
- unit_tests/sources/fixtures/__init__.py +0 -3
- unit_tests/sources/fixtures/source_test_fixture.py +0 -153
- unit_tests/sources/message/__init__.py +0 -0
- unit_tests/sources/message/test_repository.py +0 -153
- unit_tests/sources/streams/__init__.py +0 -0
- unit_tests/sources/streams/concurrent/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
- unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
- unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
- unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
- unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
- unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
- unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
- unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
- unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
- unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
- unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
- unit_tests/sources/streams/http/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/test_auth.py +0 -173
- unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
- unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
- unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
- unit_tests/sources/streams/http/test_http.py +0 -635
- unit_tests/sources/streams/test_availability_strategy.py +0 -70
- unit_tests/sources/streams/test_call_rate.py +0 -300
- unit_tests/sources/streams/test_stream_read.py +0 -405
- unit_tests/sources/streams/test_streams_core.py +0 -184
- unit_tests/sources/test_abstract_source.py +0 -1442
- unit_tests/sources/test_concurrent_source.py +0 -112
- unit_tests/sources/test_config.py +0 -92
- unit_tests/sources/test_connector_state_manager.py +0 -482
- unit_tests/sources/test_http_logger.py +0 -252
- unit_tests/sources/test_integration_source.py +0 -86
- unit_tests/sources/test_source.py +0 -684
- unit_tests/sources/test_source_read.py +0 -460
- unit_tests/test/__init__.py +0 -0
- unit_tests/test/mock_http/__init__.py +0 -0
- unit_tests/test/mock_http/test_matcher.py +0 -53
- unit_tests/test/mock_http/test_mocker.py +0 -214
- unit_tests/test/mock_http/test_request.py +0 -117
- unit_tests/test/mock_http/test_response_builder.py +0 -177
- unit_tests/test/test_entrypoint_wrapper.py +0 -240
- unit_tests/utils/__init__.py +0 -0
- unit_tests/utils/test_datetime_format_inferrer.py +0 -60
- unit_tests/utils/test_mapping_helpers.py +0 -54
- unit_tests/utils/test_message_utils.py +0 -91
- unit_tests/utils/test_rate_limiting.py +0 -26
- unit_tests/utils/test_schema_inferrer.py +0 -202
- unit_tests/utils/test_secret_utils.py +0 -135
- unit_tests/utils/test_stream_status_utils.py +0 -61
- unit_tests/utils/test_traced_exception.py +0 -107
- /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
- {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
- {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
- {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
- {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.17.1.dev0.dist-info}/LICENSE.txt +0 -0
airbyte_cdk/sql/types.py
ADDED
@@ -0,0 +1,160 @@
|
|
1
|
+
# noqa: A005 # Allow shadowing the built-in 'types' module
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
+
|
4
|
+
"""Type conversion methods for SQL Caches."""
|
5
|
+
|
6
|
+
from __future__ import annotations
|
7
|
+
|
8
|
+
from typing import Any, cast
|
9
|
+
|
10
|
+
import sqlalchemy
|
11
|
+
|
12
|
+
# Compare to documentation here: https://docs.airbyte.com/understanding-airbyte/supported-data-types
|
13
|
+
CONVERSION_MAP = {
|
14
|
+
"string": sqlalchemy.types.VARCHAR,
|
15
|
+
"integer": sqlalchemy.types.BIGINT,
|
16
|
+
"number": sqlalchemy.types.DECIMAL(38, 9),
|
17
|
+
"boolean": sqlalchemy.types.BOOLEAN,
|
18
|
+
"date": sqlalchemy.types.DATE,
|
19
|
+
"timestamp_with_timezone": sqlalchemy.types.TIMESTAMP,
|
20
|
+
"timestamp_without_timezone": sqlalchemy.types.TIMESTAMP,
|
21
|
+
"time_with_timezone": sqlalchemy.types.TIME,
|
22
|
+
"time_without_timezone": sqlalchemy.types.TIME,
|
23
|
+
# Technically 'object' and 'array' as JSON Schema types, not airbyte types.
|
24
|
+
# We include them here for completeness.
|
25
|
+
"object": sqlalchemy.types.JSON,
|
26
|
+
"array": sqlalchemy.types.JSON,
|
27
|
+
"vector_array": sqlalchemy.types.ARRAY,
|
28
|
+
}
|
29
|
+
|
30
|
+
|
31
|
+
class SQLTypeConversionError(Exception):
|
32
|
+
"""An exception to be raised when a type conversion fails."""
|
33
|
+
|
34
|
+
|
35
|
+
def _get_airbyte_type( # noqa: PLR0911 # Too many return statements
|
36
|
+
json_schema_property_def: dict[str, str | dict[str, Any] | list[Any]],
|
37
|
+
) -> tuple[str, str | None]:
|
38
|
+
"""Get the airbyte type and subtype from a JSON schema property definition.
|
39
|
+
|
40
|
+
Subtype is only used for array types. Otherwise, subtype will return None.
|
41
|
+
"""
|
42
|
+
airbyte_type = cast(str, json_schema_property_def.get("airbyte_type", None))
|
43
|
+
if airbyte_type:
|
44
|
+
return airbyte_type, None
|
45
|
+
|
46
|
+
json_schema_type = json_schema_property_def.get("type", None)
|
47
|
+
json_schema_format = json_schema_property_def.get("format", None)
|
48
|
+
|
49
|
+
# if json_schema_type is an array of two strings with one of them being null, pick the other one
|
50
|
+
# this strategy is often used by connectors to indicate a field might not be set all the time
|
51
|
+
if isinstance(json_schema_type, list):
|
52
|
+
non_null_types = [t for t in json_schema_type if t != "null"]
|
53
|
+
if len(non_null_types) == 1:
|
54
|
+
json_schema_type = non_null_types[0]
|
55
|
+
|
56
|
+
if json_schema_type == "string":
|
57
|
+
if json_schema_format == "date":
|
58
|
+
return "date", None
|
59
|
+
|
60
|
+
if json_schema_format == "date-time":
|
61
|
+
return "timestamp_with_timezone", None
|
62
|
+
|
63
|
+
if json_schema_format == "time":
|
64
|
+
return "time_without_timezone", None
|
65
|
+
|
66
|
+
if isinstance(json_schema_type, str) and json_schema_type in {
|
67
|
+
"string",
|
68
|
+
"number",
|
69
|
+
"boolean",
|
70
|
+
"integer",
|
71
|
+
}:
|
72
|
+
return json_schema_type, None
|
73
|
+
|
74
|
+
if json_schema_type == "object":
|
75
|
+
return "object", None
|
76
|
+
|
77
|
+
if json_schema_type == "array":
|
78
|
+
items_def = json_schema_property_def.get("items", None)
|
79
|
+
if isinstance(items_def, dict):
|
80
|
+
try:
|
81
|
+
subtype, _ = _get_airbyte_type(items_def)
|
82
|
+
except SQLTypeConversionError:
|
83
|
+
# We have enough information, so we can ignore parsing errors on subtype.
|
84
|
+
subtype = None
|
85
|
+
|
86
|
+
return "array", subtype
|
87
|
+
|
88
|
+
return "array", None
|
89
|
+
|
90
|
+
if json_schema_type == "vector_array":
|
91
|
+
return "vector_array", "Float"
|
92
|
+
|
93
|
+
err_msg = f"Could not determine airbyte type from JSON schema type: {json_schema_property_def}"
|
94
|
+
raise SQLTypeConversionError(err_msg)
|
95
|
+
|
96
|
+
|
97
|
+
class SQLTypeConverter:
|
98
|
+
"""A base class to perform type conversions."""
|
99
|
+
|
100
|
+
def __init__(
|
101
|
+
self,
|
102
|
+
conversion_map: dict[str, Any] | None = None,
|
103
|
+
) -> None:
|
104
|
+
"""Initialize the type converter."""
|
105
|
+
self.conversion_map = conversion_map or CONVERSION_MAP
|
106
|
+
|
107
|
+
@classmethod
|
108
|
+
def get_string_type(cls) -> sqlalchemy.types.TypeEngine[str]:
|
109
|
+
"""Get the type to use for string data."""
|
110
|
+
return sqlalchemy.types.VARCHAR()
|
111
|
+
|
112
|
+
@classmethod
|
113
|
+
def get_failover_type(cls) -> sqlalchemy.types.TypeEngine[str]:
|
114
|
+
"""Get the 'last resort' type to use if no other type is found."""
|
115
|
+
return cls.get_string_type()
|
116
|
+
|
117
|
+
@classmethod
|
118
|
+
def get_json_type(cls) -> sqlalchemy.types.TypeEngine[Any]:
|
119
|
+
"""Get the type to use for nested JSON data."""
|
120
|
+
return sqlalchemy.types.JSON()
|
121
|
+
|
122
|
+
def to_sql_type( # noqa: PLR0911 # Too many return statements
|
123
|
+
self,
|
124
|
+
json_schema_property_def: dict[str, str | dict[str, Any] | list[Any]],
|
125
|
+
) -> Any:
|
126
|
+
"""Convert a value to a SQL type."""
|
127
|
+
try:
|
128
|
+
airbyte_type, _ = _get_airbyte_type(json_schema_property_def)
|
129
|
+
# to-do - is there a better way to check the following
|
130
|
+
if airbyte_type == "vector_array":
|
131
|
+
return sqlalchemy.types.ARRAY(sqlalchemy.types.Float())
|
132
|
+
sql_type = self.conversion_map[airbyte_type]
|
133
|
+
except SQLTypeConversionError:
|
134
|
+
print(f"Could not determine airbyte type from JSON schema: {json_schema_property_def}")
|
135
|
+
except KeyError:
|
136
|
+
print(f"Could not find SQL type for airbyte type: {airbyte_type}")
|
137
|
+
else:
|
138
|
+
# No exceptions were raised, so we can return the SQL type.
|
139
|
+
if isinstance(sql_type, type):
|
140
|
+
# This is a class. Call its constructor.
|
141
|
+
sql_type = sql_type()
|
142
|
+
|
143
|
+
return sql_type
|
144
|
+
|
145
|
+
json_schema_type = json_schema_property_def.get("type", None)
|
146
|
+
json_schema_format = json_schema_property_def.get("format", None)
|
147
|
+
|
148
|
+
if json_schema_type == "string" and json_schema_format == "date":
|
149
|
+
return sqlalchemy.types.DATE()
|
150
|
+
|
151
|
+
if json_schema_type == "string" and json_schema_format == "date-time":
|
152
|
+
return sqlalchemy.types.TIMESTAMP()
|
153
|
+
|
154
|
+
if json_schema_type == "array":
|
155
|
+
return sqlalchemy.types.JSON()
|
156
|
+
|
157
|
+
if json_schema_type == "object":
|
158
|
+
return sqlalchemy.types.JSON()
|
159
|
+
|
160
|
+
return self.get_failover_type()
|
@@ -1,29 +1,81 @@
|
|
1
1
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
2
2
|
|
3
|
-
from typing import Any, Dict, List
|
3
|
+
from typing import Any, Dict, List, Union, overload
|
4
4
|
|
5
|
-
from
|
5
|
+
from airbyte_cdk.models import (
|
6
|
+
ConfiguredAirbyteCatalog,
|
7
|
+
ConfiguredAirbyteStream,
|
8
|
+
ConfiguredAirbyteStreamSerializer,
|
9
|
+
SyncMode,
|
10
|
+
)
|
11
|
+
|
12
|
+
|
13
|
+
class ConfiguredAirbyteStreamBuilder:
|
14
|
+
def __init__(self) -> None:
|
15
|
+
self._stream: Dict[str, Any] = {
|
16
|
+
"stream": {
|
17
|
+
"name": "any name",
|
18
|
+
"json_schema": {},
|
19
|
+
"supported_sync_modes": ["full_refresh", "incremental"],
|
20
|
+
"source_defined_primary_key": [["id"]],
|
21
|
+
},
|
22
|
+
"primary_key": [["id"]],
|
23
|
+
"sync_mode": "full_refresh",
|
24
|
+
"destination_sync_mode": "overwrite",
|
25
|
+
}
|
26
|
+
|
27
|
+
def with_name(self, name: str) -> "ConfiguredAirbyteStreamBuilder":
|
28
|
+
self._stream["stream"]["name"] = name # type: ignore # we assume that self._stream["stream"] is a Dict[str, Any]
|
29
|
+
return self
|
30
|
+
|
31
|
+
def with_sync_mode(self, sync_mode: SyncMode) -> "ConfiguredAirbyteStreamBuilder":
|
32
|
+
self._stream["sync_mode"] = sync_mode.name
|
33
|
+
return self
|
34
|
+
|
35
|
+
def with_primary_key(self, pk: List[List[str]]) -> "ConfiguredAirbyteStreamBuilder":
|
36
|
+
self._stream["primary_key"] = pk
|
37
|
+
self._stream["stream"]["source_defined_primary_key"] = pk # type: ignore # we assume that self._stream["stream"] is a Dict[str, Any]
|
38
|
+
return self
|
39
|
+
|
40
|
+
def with_json_schema(self, json_schema: Dict[str, Any]) -> "ConfiguredAirbyteStreamBuilder":
|
41
|
+
self._stream["stream"]["json_schema"] = json_schema
|
42
|
+
return self
|
43
|
+
|
44
|
+
def build(self) -> ConfiguredAirbyteStream:
|
45
|
+
return ConfiguredAirbyteStreamSerializer.load(self._stream)
|
6
46
|
|
7
47
|
|
8
48
|
class CatalogBuilder:
|
9
49
|
def __init__(self) -> None:
|
10
|
-
self._streams: List[
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
50
|
+
self._streams: List[ConfiguredAirbyteStreamBuilder] = []
|
51
|
+
|
52
|
+
@overload
|
53
|
+
def with_stream(self, name: ConfiguredAirbyteStreamBuilder) -> "CatalogBuilder": ...
|
54
|
+
|
55
|
+
@overload
|
56
|
+
def with_stream(self, name: str, sync_mode: SyncMode) -> "CatalogBuilder": ...
|
57
|
+
|
58
|
+
def with_stream(
|
59
|
+
self,
|
60
|
+
name: Union[str, ConfiguredAirbyteStreamBuilder],
|
61
|
+
sync_mode: Union[SyncMode, None] = None,
|
62
|
+
) -> "CatalogBuilder":
|
63
|
+
# As we are introducing a fully fledge ConfiguredAirbyteStreamBuilder, we would like to deprecate the previous interface
|
64
|
+
# with_stream(str, SyncMode)
|
65
|
+
|
66
|
+
# to avoid a breaking change, `name` needs to stay in the API but this can be either a name or a builder
|
67
|
+
name_or_builder = name
|
68
|
+
builder = (
|
69
|
+
name_or_builder
|
70
|
+
if isinstance(name_or_builder, ConfiguredAirbyteStreamBuilder)
|
71
|
+
else ConfiguredAirbyteStreamBuilder()
|
72
|
+
.with_name(name_or_builder)
|
73
|
+
.with_sync_mode(sync_mode)
|
25
74
|
)
|
75
|
+
self._streams.append(builder)
|
26
76
|
return self
|
27
77
|
|
28
78
|
def build(self) -> ConfiguredAirbyteCatalog:
|
29
|
-
return ConfiguredAirbyteCatalog
|
79
|
+
return ConfiguredAirbyteCatalog(
|
80
|
+
streams=list(map(lambda builder: builder.build(), self._streams))
|
81
|
+
)
|
@@ -16,46 +16,59 @@ than that, there are integrations point that are annoying to integrate with usin
|
|
16
16
|
|
17
17
|
import json
|
18
18
|
import logging
|
19
|
+
import re
|
19
20
|
import tempfile
|
20
21
|
import traceback
|
21
22
|
from io import StringIO
|
22
23
|
from pathlib import Path
|
23
24
|
from typing import Any, List, Mapping, Optional, Union
|
24
25
|
|
26
|
+
import orjson
|
27
|
+
from pydantic import ValidationError as V2ValidationError
|
28
|
+
from serpyco_rs import SchemaValidationError
|
29
|
+
|
25
30
|
from airbyte_cdk.entrypoint import AirbyteEntrypoint
|
26
31
|
from airbyte_cdk.exception_handler import assemble_uncaught_exception
|
27
32
|
from airbyte_cdk.logger import AirbyteLogFormatter
|
28
|
-
from airbyte_cdk.
|
29
|
-
from airbyte_protocol.models import (
|
33
|
+
from airbyte_cdk.models import (
|
30
34
|
AirbyteLogMessage,
|
31
35
|
AirbyteMessage,
|
36
|
+
AirbyteMessageSerializer,
|
32
37
|
AirbyteStateMessage,
|
38
|
+
AirbyteStateMessageSerializer,
|
33
39
|
AirbyteStreamStatus,
|
34
40
|
ConfiguredAirbyteCatalog,
|
41
|
+
ConfiguredAirbyteCatalogSerializer,
|
35
42
|
Level,
|
36
43
|
TraceType,
|
37
44
|
Type,
|
38
45
|
)
|
39
|
-
from
|
46
|
+
from airbyte_cdk.sources import Source
|
40
47
|
|
41
48
|
|
42
49
|
class EntrypointOutput:
|
43
50
|
def __init__(self, messages: List[str], uncaught_exception: Optional[BaseException] = None):
|
44
51
|
try:
|
45
52
|
self._messages = [self._parse_message(message) for message in messages]
|
46
|
-
except
|
53
|
+
except V2ValidationError as exception:
|
47
54
|
raise ValueError("All messages are expected to be AirbyteMessage") from exception
|
48
55
|
|
49
56
|
if uncaught_exception:
|
50
|
-
self._messages.append(
|
57
|
+
self._messages.append(
|
58
|
+
assemble_uncaught_exception(
|
59
|
+
type(uncaught_exception), uncaught_exception
|
60
|
+
).as_airbyte_message()
|
61
|
+
)
|
51
62
|
|
52
63
|
@staticmethod
|
53
64
|
def _parse_message(message: str) -> AirbyteMessage:
|
54
65
|
try:
|
55
|
-
return
|
56
|
-
except (
|
66
|
+
return AirbyteMessageSerializer.load(orjson.loads(message))
|
67
|
+
except (orjson.JSONDecodeError, SchemaValidationError):
|
57
68
|
# The platform assumes that logs that are not of AirbyteMessage format are log messages
|
58
|
-
return AirbyteMessage(
|
69
|
+
return AirbyteMessage(
|
70
|
+
type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message=message)
|
71
|
+
)
|
59
72
|
|
60
73
|
@property
|
61
74
|
def records_and_state_messages(self) -> List[AirbyteMessage]:
|
@@ -74,7 +87,7 @@ class EntrypointOutput:
|
|
74
87
|
state_messages = self._get_message_by_types([Type.STATE])
|
75
88
|
if not state_messages:
|
76
89
|
raise ValueError("Can't provide most recent state as there are no state messages")
|
77
|
-
return state_messages[-1].state.stream
|
90
|
+
return state_messages[-1].state.stream # type: ignore[union-attr] # state has `stream`
|
78
91
|
|
79
92
|
@property
|
80
93
|
def logs(self) -> List[AirbyteMessage]:
|
@@ -92,11 +105,18 @@ class EntrypointOutput:
|
|
92
105
|
def errors(self) -> List[AirbyteMessage]:
|
93
106
|
return self._get_trace_message_by_trace_type(TraceType.ERROR)
|
94
107
|
|
108
|
+
@property
|
109
|
+
def catalog(self) -> AirbyteMessage:
|
110
|
+
catalog = self._get_message_by_types([Type.CATALOG])
|
111
|
+
if len(catalog) != 1:
|
112
|
+
raise ValueError(f"Expected exactly one catalog but got {len(catalog)}")
|
113
|
+
return catalog[0]
|
114
|
+
|
95
115
|
def get_stream_statuses(self, stream_name: str) -> List[AirbyteStreamStatus]:
|
96
116
|
status_messages = map(
|
97
|
-
lambda message: message.trace.stream_status.status,
|
117
|
+
lambda message: message.trace.stream_status.status, # type: ignore
|
98
118
|
filter(
|
99
|
-
lambda message: message.trace.stream_status.stream_descriptor.name == stream_name,
|
119
|
+
lambda message: message.trace.stream_status.stream_descriptor.name == stream_name, # type: ignore # callable; trace has `stream_status`
|
100
120
|
self._get_trace_message_by_trace_type(TraceType.STREAM_STATUS),
|
101
121
|
),
|
102
122
|
)
|
@@ -106,7 +126,77 @@ class EntrypointOutput:
|
|
106
126
|
return [message for message in self._messages if message.type in message_types]
|
107
127
|
|
108
128
|
def _get_trace_message_by_trace_type(self, trace_type: TraceType) -> List[AirbyteMessage]:
|
109
|
-
return [
|
129
|
+
return [
|
130
|
+
message
|
131
|
+
for message in self._get_message_by_types([Type.TRACE])
|
132
|
+
if message.trace.type == trace_type # type: ignore[union-attr] # trace has `type`
|
133
|
+
]
|
134
|
+
|
135
|
+
def is_in_logs(self, pattern: str) -> bool:
|
136
|
+
"""Check if any log message case-insensitive matches the pattern."""
|
137
|
+
return any(
|
138
|
+
re.search(
|
139
|
+
pattern,
|
140
|
+
entry.log.message, # type: ignore[union-attr] # log has `message`
|
141
|
+
flags=re.IGNORECASE,
|
142
|
+
)
|
143
|
+
for entry in self.logs
|
144
|
+
)
|
145
|
+
|
146
|
+
def is_not_in_logs(self, pattern: str) -> bool:
|
147
|
+
"""Check if no log message matches the case-insensitive pattern."""
|
148
|
+
return not self.is_in_logs(pattern)
|
149
|
+
|
150
|
+
|
151
|
+
def _run_command(
|
152
|
+
source: Source, args: List[str], expecting_exception: bool = False
|
153
|
+
) -> EntrypointOutput:
|
154
|
+
log_capture_buffer = StringIO()
|
155
|
+
stream_handler = logging.StreamHandler(log_capture_buffer)
|
156
|
+
stream_handler.setLevel(logging.INFO)
|
157
|
+
stream_handler.setFormatter(AirbyteLogFormatter())
|
158
|
+
parent_logger = logging.getLogger("")
|
159
|
+
parent_logger.addHandler(stream_handler)
|
160
|
+
|
161
|
+
parsed_args = AirbyteEntrypoint.parse_args(args)
|
162
|
+
|
163
|
+
source_entrypoint = AirbyteEntrypoint(source)
|
164
|
+
messages = []
|
165
|
+
uncaught_exception = None
|
166
|
+
try:
|
167
|
+
for message in source_entrypoint.run(parsed_args):
|
168
|
+
messages.append(message)
|
169
|
+
except Exception as exception:
|
170
|
+
if not expecting_exception:
|
171
|
+
print("Printing unexpected error from entrypoint_wrapper")
|
172
|
+
print("".join(traceback.format_exception(None, exception, exception.__traceback__)))
|
173
|
+
uncaught_exception = exception
|
174
|
+
|
175
|
+
captured_logs = log_capture_buffer.getvalue().split("\n")[:-1]
|
176
|
+
|
177
|
+
parent_logger.removeHandler(stream_handler)
|
178
|
+
|
179
|
+
return EntrypointOutput(messages + captured_logs, uncaught_exception)
|
180
|
+
|
181
|
+
|
182
|
+
def discover(
|
183
|
+
source: Source,
|
184
|
+
config: Mapping[str, Any],
|
185
|
+
expecting_exception: bool = False,
|
186
|
+
) -> EntrypointOutput:
|
187
|
+
"""
|
188
|
+
config must be json serializable
|
189
|
+
:param expecting_exception: By default if there is an uncaught exception, the exception will be printed out. If this is expected, please
|
190
|
+
provide expecting_exception=True so that the test output logs are cleaner
|
191
|
+
"""
|
192
|
+
|
193
|
+
with tempfile.TemporaryDirectory() as tmp_directory:
|
194
|
+
tmp_directory_path = Path(tmp_directory)
|
195
|
+
config_file = make_file(tmp_directory_path / "config.json", config)
|
196
|
+
|
197
|
+
return _run_command(
|
198
|
+
source, ["discover", "--config", config_file, "--debug"], expecting_exception
|
199
|
+
)
|
110
200
|
|
111
201
|
|
112
202
|
def read(
|
@@ -122,52 +212,37 @@ def read(
|
|
122
212
|
:param expecting_exception: By default if there is an uncaught exception, the exception will be printed out. If this is expected, please
|
123
213
|
provide expecting_exception=True so that the test output logs are cleaner
|
124
214
|
"""
|
125
|
-
log_capture_buffer = StringIO()
|
126
|
-
stream_handler = logging.StreamHandler(log_capture_buffer)
|
127
|
-
stream_handler.setLevel(logging.INFO)
|
128
|
-
stream_handler.setFormatter(AirbyteLogFormatter())
|
129
|
-
parent_logger = logging.getLogger("")
|
130
|
-
parent_logger.addHandler(stream_handler)
|
131
|
-
|
132
215
|
with tempfile.TemporaryDirectory() as tmp_directory:
|
133
216
|
tmp_directory_path = Path(tmp_directory)
|
217
|
+
config_file = make_file(tmp_directory_path / "config.json", config)
|
218
|
+
catalog_file = make_file(
|
219
|
+
tmp_directory_path / "catalog.json",
|
220
|
+
orjson.dumps(ConfiguredAirbyteCatalogSerializer.dump(catalog)).decode(),
|
221
|
+
)
|
134
222
|
args = [
|
135
223
|
"read",
|
136
224
|
"--config",
|
137
|
-
|
225
|
+
config_file,
|
138
226
|
"--catalog",
|
139
|
-
|
227
|
+
catalog_file,
|
140
228
|
]
|
141
229
|
if state is not None:
|
142
230
|
args.extend(
|
143
231
|
[
|
144
232
|
"--state",
|
145
|
-
make_file(
|
233
|
+
make_file(
|
234
|
+
tmp_directory_path / "state.json",
|
235
|
+
f"[{','.join([orjson.dumps(AirbyteStateMessageSerializer.dump(stream_state)).decode() for stream_state in state])}]",
|
236
|
+
),
|
146
237
|
]
|
147
238
|
)
|
148
|
-
args.append("--debug")
|
149
|
-
source_entrypoint = AirbyteEntrypoint(source)
|
150
|
-
parsed_args = source_entrypoint.parse_args(args)
|
151
|
-
|
152
|
-
messages = []
|
153
|
-
uncaught_exception = None
|
154
|
-
try:
|
155
|
-
for message in source_entrypoint.run(parsed_args):
|
156
|
-
messages.append(message)
|
157
|
-
except Exception as exception:
|
158
|
-
if not expecting_exception:
|
159
|
-
print("Printing unexpected error from entrypoint_wrapper")
|
160
|
-
print("".join(traceback.format_exception(None, exception, exception.__traceback__)))
|
161
|
-
uncaught_exception = exception
|
162
|
-
|
163
|
-
captured_logs = log_capture_buffer.getvalue().split("\n")[:-1]
|
164
|
-
|
165
|
-
parent_logger.removeHandler(stream_handler)
|
166
239
|
|
167
|
-
return
|
240
|
+
return _run_command(source, args, expecting_exception)
|
168
241
|
|
169
242
|
|
170
|
-
def make_file(
|
243
|
+
def make_file(
|
244
|
+
path: Path, file_contents: Optional[Union[str, Mapping[str, Any], List[Mapping[str, Any]]]]
|
245
|
+
) -> str:
|
171
246
|
if isinstance(file_contents, str):
|
172
247
|
path.write_text(file_contents)
|
173
248
|
else:
|
@@ -1,6 +1,6 @@
|
|
1
1
|
from airbyte_cdk.test.mock_http.matcher import HttpRequestMatcher
|
2
|
+
from airbyte_cdk.test.mock_http.mocker import HttpMocker
|
2
3
|
from airbyte_cdk.test.mock_http.request import HttpRequest
|
3
4
|
from airbyte_cdk.test.mock_http.response import HttpResponse
|
4
|
-
from airbyte_cdk.test.mock_http.mocker import HttpMocker
|
5
5
|
|
6
6
|
__all__ = ["HttpMocker", "HttpRequest", "HttpRequestMatcher", "HttpResponse"]
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
2
|
+
from typing import Any
|
2
3
|
|
3
4
|
from airbyte_cdk.test.mock_http.request import HttpRequest
|
4
5
|
|
@@ -33,3 +34,8 @@ class HttpRequestMatcher:
|
|
33
34
|
f"minimum_number_of_expected_match={self._minimum_number_of_expected_match}, "
|
34
35
|
f"actual_number_of_matches={self._actual_number_of_matches})"
|
35
36
|
)
|
37
|
+
|
38
|
+
def __eq__(self, other: Any) -> bool:
|
39
|
+
if isinstance(other, HttpRequestMatcher):
|
40
|
+
return self._request_to_match == other._request_to_match
|
41
|
+
return False
|
@@ -7,12 +7,17 @@ from types import TracebackType
|
|
7
7
|
from typing import Callable, List, Optional, Union
|
8
8
|
|
9
9
|
import requests_mock
|
10
|
-
|
10
|
+
|
11
|
+
from airbyte_cdk.test.mock_http.matcher import HttpRequestMatcher
|
12
|
+
from airbyte_cdk.test.mock_http.request import HttpRequest
|
13
|
+
from airbyte_cdk.test.mock_http.response import HttpResponse
|
11
14
|
|
12
15
|
|
13
16
|
class SupportedHttpMethods(str, Enum):
|
14
17
|
GET = "get"
|
18
|
+
PATCH = "patch"
|
15
19
|
POST = "post"
|
20
|
+
DELETE = "delete"
|
16
21
|
|
17
22
|
|
18
23
|
class HttpMocker(contextlib.ContextDecorator):
|
@@ -40,7 +45,12 @@ class HttpMocker(contextlib.ContextDecorator):
|
|
40
45
|
self._mocker.__enter__()
|
41
46
|
return self
|
42
47
|
|
43
|
-
def __exit__(
|
48
|
+
def __exit__(
|
49
|
+
self,
|
50
|
+
exc_type: Optional[BaseException],
|
51
|
+
exc_val: Optional[BaseException],
|
52
|
+
exc_tb: Optional[TracebackType],
|
53
|
+
) -> None:
|
44
54
|
self._mocker.__exit__(exc_type, exc_val, exc_tb)
|
45
55
|
|
46
56
|
def _validate_all_matchers_called(self) -> None:
|
@@ -49,43 +59,74 @@ class HttpMocker(contextlib.ContextDecorator):
|
|
49
59
|
raise ValueError(f"Invalid number of matches for `{matcher}`")
|
50
60
|
|
51
61
|
def _mock_request_method(
|
52
|
-
self,
|
62
|
+
self,
|
63
|
+
method: SupportedHttpMethods,
|
64
|
+
request: HttpRequest,
|
65
|
+
responses: Union[HttpResponse, List[HttpResponse]],
|
53
66
|
) -> None:
|
54
67
|
if isinstance(responses, HttpResponse):
|
55
68
|
responses = [responses]
|
56
69
|
|
57
70
|
matcher = HttpRequestMatcher(request, len(responses))
|
71
|
+
if matcher in self._matchers:
|
72
|
+
raise ValueError(f"Request {matcher.request} already mocked")
|
58
73
|
self._matchers.append(matcher)
|
59
74
|
|
60
75
|
getattr(self._mocker, method)(
|
61
76
|
requests_mock.ANY,
|
62
77
|
additional_matcher=self._matches_wrapper(matcher),
|
63
78
|
response_list=[
|
64
|
-
{
|
79
|
+
{
|
80
|
+
"text": response.body,
|
81
|
+
"status_code": response.status_code,
|
82
|
+
"headers": response.headers,
|
83
|
+
}
|
84
|
+
for response in responses
|
65
85
|
],
|
66
86
|
)
|
67
87
|
|
68
88
|
def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
|
69
89
|
self._mock_request_method(SupportedHttpMethods.GET, request, responses)
|
70
90
|
|
71
|
-
def
|
91
|
+
def patch(
|
92
|
+
self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
|
93
|
+
) -> None:
|
94
|
+
self._mock_request_method(SupportedHttpMethods.PATCH, request, responses)
|
95
|
+
|
96
|
+
def post(
|
97
|
+
self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
|
98
|
+
) -> None:
|
72
99
|
self._mock_request_method(SupportedHttpMethods.POST, request, responses)
|
73
100
|
|
101
|
+
def delete(
|
102
|
+
self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
|
103
|
+
) -> None:
|
104
|
+
self._mock_request_method(SupportedHttpMethods.DELETE, request, responses)
|
105
|
+
|
74
106
|
@staticmethod
|
75
|
-
def _matches_wrapper(
|
107
|
+
def _matches_wrapper(
|
108
|
+
matcher: HttpRequestMatcher,
|
109
|
+
) -> Callable[[requests_mock.request._RequestObjectProxy], bool]:
|
76
110
|
def matches(requests_mock_request: requests_mock.request._RequestObjectProxy) -> bool:
|
77
111
|
# query_params are provided as part of `requests_mock_request.url`
|
78
112
|
http_request = HttpRequest(
|
79
|
-
requests_mock_request.url,
|
113
|
+
requests_mock_request.url,
|
114
|
+
query_params={},
|
115
|
+
headers=requests_mock_request.headers,
|
116
|
+
body=requests_mock_request.body,
|
80
117
|
)
|
81
118
|
return matcher.matches(http_request)
|
82
119
|
|
83
120
|
return matches
|
84
121
|
|
85
122
|
def assert_number_of_calls(self, request: HttpRequest, number_of_calls: int) -> None:
|
86
|
-
corresponding_matchers = list(
|
123
|
+
corresponding_matchers = list(
|
124
|
+
filter(lambda matcher: matcher.request == request, self._matchers)
|
125
|
+
)
|
87
126
|
if len(corresponding_matchers) != 1:
|
88
|
-
raise ValueError(
|
127
|
+
raise ValueError(
|
128
|
+
f"Was expecting only one matcher to match the request but got `{corresponding_matchers}`"
|
129
|
+
)
|
89
130
|
|
90
131
|
assert corresponding_matchers[0].actual_number_of_matches == number_of_calls
|
91
132
|
|
@@ -100,7 +141,9 @@ class HttpMocker(contextlib.ContextDecorator):
|
|
100
141
|
try:
|
101
142
|
result = f(*args, **kwargs)
|
102
143
|
except requests_mock.NoMockAddress as no_mock_exception:
|
103
|
-
matchers_as_string = "\n\t".join(
|
144
|
+
matchers_as_string = "\n\t".join(
|
145
|
+
map(lambda matcher: str(matcher.request), self._matchers)
|
146
|
+
)
|
104
147
|
raise ValueError(
|
105
148
|
f"No matcher matches {no_mock_exception.args[0]} with headers `{no_mock_exception.request.headers}` "
|
106
149
|
f"and body `{no_mock_exception.request.body}`. "
|
@@ -123,3 +166,7 @@ class HttpMocker(contextlib.ContextDecorator):
|
|
123
166
|
return result
|
124
167
|
|
125
168
|
return wrapper
|
169
|
+
|
170
|
+
def clear_all_matchers(self) -> None:
|
171
|
+
"""Clears all stored matchers by resetting the _matchers list to an empty state."""
|
172
|
+
self._matchers = []
|