airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- airbyte_cdk/__init__.py +355 -6
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +29 -10
- airbyte_cdk/connector.py +24 -24
- airbyte_cdk/connector_builder/README.md +53 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
- airbyte_cdk/connector_builder/main.py +45 -13
- airbyte_cdk/connector_builder/message_grouper.py +189 -50
- airbyte_cdk/connector_builder/models.py +3 -2
- airbyte_cdk/destinations/__init__.py +4 -3
- airbyte_cdk/destinations/destination.py +54 -20
- airbyte_cdk/destinations/vector_db_based/README.md +37 -0
- airbyte_cdk/destinations/vector_db_based/config.py +40 -17
- airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
- airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
- airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
- airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
- airbyte_cdk/entrypoint.py +153 -44
- airbyte_cdk/exception_handler.py +21 -3
- airbyte_cdk/logger.py +30 -44
- airbyte_cdk/models/__init__.py +13 -2
- airbyte_cdk/models/airbyte_protocol.py +86 -1
- airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
- airbyte_cdk/models/file_transfer_record_message.py +13 -0
- airbyte_cdk/models/well_known_types.py +1 -1
- airbyte_cdk/sources/__init__.py +5 -1
- airbyte_cdk/sources/abstract_source.py +125 -79
- airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
- airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
- airbyte_cdk/sources/config.py +3 -2
- airbyte_cdk/sources/connector_state_manager.py +49 -83
- airbyte_cdk/sources/declarative/async_job/job.py +52 -0
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
- airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
- airbyte_cdk/sources/declarative/async_job/status.py +24 -0
- airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
- airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
- airbyte_cdk/sources/declarative/auth/token.py +28 -10
- airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
- airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
- airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
- airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +421 -0
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1213 -88
- airbyte_cdk/sources/declarative/declarative_source.py +5 -2
- airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
- airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
- airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
- airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
- airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
- airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
- airbyte_cdk/sources/declarative/extractors/record_filter.py +65 -8
- airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
- airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
- airbyte_cdk/sources/declarative/incremental/__init__.py +25 -3
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
- airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +159 -74
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
- airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
- airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
- airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
- airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
- airbyte_cdk/sources/declarative/models/__init__.py +1 -1
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1329 -595
- airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1699 -226
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
- airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
- airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
- airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
- airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
- airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
- airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
- airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
- airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
- airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
- airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
- airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +228 -72
- airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
- airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
- airbyte_cdk/sources/declarative/spec/spec.py +12 -5
- airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
- airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
- airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
- airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
- airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
- airbyte_cdk/sources/declarative/types.py +19 -110
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
- airbyte_cdk/sources/embedded/base_integration.py +16 -5
- airbyte_cdk/sources/embedded/catalog.py +16 -4
- airbyte_cdk/sources/embedded/runner.py +19 -3
- airbyte_cdk/sources/embedded/tools.py +5 -2
- airbyte_cdk/sources/file_based/README.md +152 -0
- airbyte_cdk/sources/file_based/__init__.py +24 -0
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +58 -10
- airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
- airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
- airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
- airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
- airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
- airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
- airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
- airbyte_cdk/sources/file_based/exceptions.py +52 -15
- airbyte_cdk/sources/file_based/file_based_source.py +163 -33
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +83 -5
- airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +145 -41
- airbyte_cdk/sources/file_based/remote_file.py +1 -1
- airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
- airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +175 -45
- airbyte_cdk/sources/http_logger.py +8 -3
- airbyte_cdk/sources/message/__init__.py +7 -1
- airbyte_cdk/sources/message/repository.py +18 -4
- airbyte_cdk/sources/source.py +42 -38
- airbyte_cdk/sources/streams/__init__.py +2 -2
- airbyte_cdk/sources/streams/availability_strategy.py +54 -3
- airbyte_cdk/sources/streams/call_rate.py +64 -21
- airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
- airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
- airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
- airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
- airbyte_cdk/sources/streams/concurrent/README.md +7 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
- airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
- airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
- airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
- airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
- airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
- airbyte_cdk/sources/streams/core.py +412 -87
- airbyte_cdk/sources/streams/http/__init__.py +2 -1
- airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
- airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
- airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
- airbyte_cdk/sources/streams/http/exceptions.py +27 -7
- airbyte_cdk/sources/streams/http/http.py +369 -246
- airbyte_cdk/sources/streams/http/http_client.py +531 -0
- airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
- airbyte_cdk/sources/types.py +154 -0
- airbyte_cdk/sources/utils/record_helper.py +36 -21
- airbyte_cdk/sources/utils/schema_helpers.py +13 -6
- airbyte_cdk/sources/utils/slice_logger.py +4 -1
- airbyte_cdk/sources/utils/transform.py +54 -20
- airbyte_cdk/sql/_util/hashing.py +34 -0
- airbyte_cdk/sql/_util/name_normalizers.py +92 -0
- airbyte_cdk/sql/constants.py +32 -0
- airbyte_cdk/sql/exceptions.py +235 -0
- airbyte_cdk/sql/secrets.py +123 -0
- airbyte_cdk/sql/shared/__init__.py +15 -0
- airbyte_cdk/sql/shared/catalog_providers.py +145 -0
- airbyte_cdk/sql/shared/sql_processor.py +786 -0
- airbyte_cdk/sql/types.py +160 -0
- airbyte_cdk/test/catalog_builder.py +70 -18
- airbyte_cdk/test/entrypoint_wrapper.py +117 -42
- airbyte_cdk/test/mock_http/__init__.py +1 -1
- airbyte_cdk/test/mock_http/matcher.py +6 -0
- airbyte_cdk/test/mock_http/mocker.py +57 -10
- airbyte_cdk/test/mock_http/request.py +19 -3
- airbyte_cdk/test/mock_http/response.py +3 -1
- airbyte_cdk/test/mock_http/response_builder.py +32 -16
- airbyte_cdk/test/state_builder.py +18 -10
- airbyte_cdk/test/utils/__init__.py +1 -0
- airbyte_cdk/test/utils/data.py +24 -0
- airbyte_cdk/test/utils/http_mocking.py +16 -0
- airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
- airbyte_cdk/test/utils/reading.py +26 -0
- airbyte_cdk/utils/__init__.py +2 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
- airbyte_cdk/utils/analytics_message.py +10 -2
- airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
- airbyte_cdk/utils/event_timing.py +10 -10
- airbyte_cdk/utils/mapping_helpers.py +3 -1
- airbyte_cdk/utils/message_utils.py +20 -11
- airbyte_cdk/utils/print_buffer.py +75 -0
- airbyte_cdk/utils/schema_inferrer.py +198 -28
- airbyte_cdk/utils/slice_hasher.py +30 -0
- airbyte_cdk/utils/spec_schema_transformations.py +6 -3
- airbyte_cdk/utils/stream_status_utils.py +8 -1
- airbyte_cdk/utils/traced_exception.py +61 -21
- airbyte_cdk-6.13.1.dev4106.dist-info/METADATA +109 -0
- airbyte_cdk-6.13.1.dev4106.dist-info/RECORD +349 -0
- {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.13.1.dev4106.dist-info}/WHEEL +1 -2
- airbyte_cdk-6.13.1.dev4106.dist-info/entry_points.txt +3 -0
- airbyte_cdk/sources/declarative/create_partial.py +0 -92
- airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
- airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
- airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
- airbyte_cdk/sources/deprecated/base_source.py +0 -94
- airbyte_cdk/sources/deprecated/client.py +0 -99
- airbyte_cdk/sources/singer/__init__.py +0 -8
- airbyte_cdk/sources/singer/singer_helpers.py +0 -304
- airbyte_cdk/sources/singer/source.py +0 -186
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
- airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
- airbyte_cdk/sources/streams/http/auth/core.py +0 -29
- airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
- airbyte_cdk/sources/streams/http/auth/token.py +0 -47
- airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
- airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
- airbyte_cdk/sources/utils/schema_models.py +0 -84
- airbyte_cdk-0.72.0.dist-info/METADATA +0 -243
- airbyte_cdk-0.72.0.dist-info/RECORD +0 -466
- airbyte_cdk-0.72.0.dist-info/top_level.txt +0 -3
- source_declarative_manifest/main.py +0 -29
- unit_tests/connector_builder/__init__.py +0 -3
- unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
- unit_tests/connector_builder/test_message_grouper.py +0 -713
- unit_tests/connector_builder/utils.py +0 -27
- unit_tests/destinations/test_destination.py +0 -243
- unit_tests/singer/test_singer_helpers.py +0 -56
- unit_tests/singer/test_singer_source.py +0 -112
- unit_tests/sources/__init__.py +0 -0
- unit_tests/sources/concurrent_source/__init__.py +0 -3
- unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
- unit_tests/sources/declarative/__init__.py +0 -3
- unit_tests/sources/declarative/auth/__init__.py +0 -3
- unit_tests/sources/declarative/auth/test_oauth.py +0 -331
- unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
- unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
- unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
- unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
- unit_tests/sources/declarative/checks/__init__.py +0 -3
- unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
- unit_tests/sources/declarative/decoders/__init__.py +0 -0
- unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
- unit_tests/sources/declarative/external_component.py +0 -13
- unit_tests/sources/declarative/extractors/__init__.py +0 -3
- unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
- unit_tests/sources/declarative/incremental/__init__.py +0 -0
- unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
- unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
- unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
- unit_tests/sources/declarative/interpolation/__init__.py +0 -3
- unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
- unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
- unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
- unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
- unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
- unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
- unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
- unit_tests/sources/declarative/parsers/__init__.py +0 -3
- unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
- unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
- unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1841
- unit_tests/sources/declarative/parsers/testing_components.py +0 -36
- unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
- unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
- unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
- unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
- unit_tests/sources/declarative/requesters/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
- unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
- unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
- unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
- unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
- unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
- unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
- unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
- unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
- unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
- unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
- unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
- unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
- unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
- unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
- unit_tests/sources/declarative/retrievers/__init__.py +0 -3
- unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
- unit_tests/sources/declarative/schema/__init__.py +0 -6
- unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
- unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
- unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
- unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
- unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
- unit_tests/sources/declarative/states/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
- unit_tests/sources/declarative/test_create_partial.py +0 -83
- unit_tests/sources/declarative/test_declarative_stream.py +0 -103
- unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
- unit_tests/sources/declarative/test_types.py +0 -39
- unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
- unit_tests/sources/file_based/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
- unit_tests/sources/file_based/config/__init__.py +0 -0
- unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
- unit_tests/sources/file_based/config/test_csv_format.py +0 -34
- unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
- unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
- unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
- unit_tests/sources/file_based/file_types/__init__.py +0 -0
- unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
- unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
- unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
- unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
- unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
- unit_tests/sources/file_based/helpers.py +0 -70
- unit_tests/sources/file_based/in_memory_files_source.py +0 -211
- unit_tests/sources/file_based/scenarios/__init__.py +0 -0
- unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
- unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
- unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
- unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
- unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
- unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
- unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
- unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
- unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
- unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
- unit_tests/sources/file_based/stream/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
- unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
- unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
- unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
- unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
- unit_tests/sources/file_based/test_scenarios.py +0 -253
- unit_tests/sources/file_based/test_schema_helpers.py +0 -346
- unit_tests/sources/fixtures/__init__.py +0 -3
- unit_tests/sources/fixtures/source_test_fixture.py +0 -153
- unit_tests/sources/message/__init__.py +0 -0
- unit_tests/sources/message/test_repository.py +0 -153
- unit_tests/sources/streams/__init__.py +0 -0
- unit_tests/sources/streams/concurrent/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
- unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
- unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
- unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
- unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
- unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
- unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
- unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
- unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
- unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
- unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
- unit_tests/sources/streams/http/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/test_auth.py +0 -173
- unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
- unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
- unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
- unit_tests/sources/streams/http/test_http.py +0 -635
- unit_tests/sources/streams/test_availability_strategy.py +0 -70
- unit_tests/sources/streams/test_call_rate.py +0 -300
- unit_tests/sources/streams/test_stream_read.py +0 -405
- unit_tests/sources/streams/test_streams_core.py +0 -184
- unit_tests/sources/test_abstract_source.py +0 -1442
- unit_tests/sources/test_concurrent_source.py +0 -112
- unit_tests/sources/test_config.py +0 -92
- unit_tests/sources/test_connector_state_manager.py +0 -482
- unit_tests/sources/test_http_logger.py +0 -252
- unit_tests/sources/test_integration_source.py +0 -86
- unit_tests/sources/test_source.py +0 -684
- unit_tests/sources/test_source_read.py +0 -460
- unit_tests/test/__init__.py +0 -0
- unit_tests/test/mock_http/__init__.py +0 -0
- unit_tests/test/mock_http/test_matcher.py +0 -53
- unit_tests/test/mock_http/test_mocker.py +0 -214
- unit_tests/test/mock_http/test_request.py +0 -117
- unit_tests/test/mock_http/test_response_builder.py +0 -177
- unit_tests/test/test_entrypoint_wrapper.py +0 -240
- unit_tests/utils/__init__.py +0 -0
- unit_tests/utils/test_datetime_format_inferrer.py +0 -60
- unit_tests/utils/test_mapping_helpers.py +0 -54
- unit_tests/utils/test_message_utils.py +0 -91
- unit_tests/utils/test_rate_limiting.py +0 -26
- unit_tests/utils/test_schema_inferrer.py +0 -202
- unit_tests/utils/test_secret_utils.py +0 -135
- unit_tests/utils/test_stream_status_utils.py +0 -61
- unit_tests/utils/test_traced_exception.py +0 -107
- /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
- {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
- {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
- {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
- {airbyte_cdk-0.72.0.dist-info → airbyte_cdk-6.13.1.dev4106.dist-info}/LICENSE.txt +0 -0
@@ -1,32 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
import pytest as pytest
|
6
|
-
from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
|
7
|
-
from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_input_provider import InterpolatedRequestInputProvider
|
8
|
-
|
9
|
-
|
10
|
-
@pytest.mark.parametrize(
|
11
|
-
"test_name, input_request_data, expected_request_data",
|
12
|
-
[
|
13
|
-
("test_static_map_data", {"a_static_request_param": "a_static_value"}, {"a_static_request_param": "a_static_value"}),
|
14
|
-
("test_map_depends_on_stream_slice", {"read_from_slice": "{{ stream_slice['slice_key'] }}"}, {"read_from_slice": "slice_value"}),
|
15
|
-
("test_map_depends_on_config", {"read_from_config": "{{ config['config_key'] }}"}, {"read_from_config": "value_of_config"}),
|
16
|
-
(
|
17
|
-
"test_map_depends_on_parameters",
|
18
|
-
{"read_from_parameters": "{{ parameters['read_from_parameters'] }}"},
|
19
|
-
{"read_from_parameters": "value_of_parameters"},
|
20
|
-
),
|
21
|
-
("test_defaults_to_empty_dictionary", None, {}),
|
22
|
-
],
|
23
|
-
)
|
24
|
-
def test_initialize_interpolated_mapping_request_input_provider(test_name, input_request_data, expected_request_data):
|
25
|
-
config = {"config_key": "value_of_config"}
|
26
|
-
stream_slice = {"slice_key": "slice_value"}
|
27
|
-
parameters = {"read_from_parameters": "value_of_parameters"}
|
28
|
-
provider = InterpolatedRequestInputProvider(request_inputs=input_request_data, config=config, parameters=parameters)
|
29
|
-
actual_request_data = provider.eval_request_inputs(stream_state={}, stream_slice=stream_slice)
|
30
|
-
|
31
|
-
assert isinstance(provider._interpolator, InterpolatedMapping)
|
32
|
-
assert actual_request_data == expected_request_data
|
@@ -1,542 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
from unittest.mock import MagicMock, Mock, patch
|
5
|
-
|
6
|
-
import pytest
|
7
|
-
import requests
|
8
|
-
from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode, Type
|
9
|
-
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth
|
10
|
-
from airbyte_cdk.sources.declarative.incremental import Cursor, DatetimeBasedCursor
|
11
|
-
from airbyte_cdk.sources.declarative.partition_routers import SinglePartitionRouter
|
12
|
-
from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus
|
13
|
-
from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType
|
14
|
-
from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
|
15
|
-
from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever, SimpleRetrieverTestReadDecorator
|
16
|
-
from airbyte_cdk.sources.declarative.types import Record
|
17
|
-
|
18
|
-
A_SLICE_STATE = {"slice_state": "slice state value"}
|
19
|
-
A_STREAM_SLICE = {"stream slice": "slice value"}
|
20
|
-
A_STREAM_STATE = {"stream state": "state value"}
|
21
|
-
|
22
|
-
primary_key = "pk"
|
23
|
-
records = [{"id": 1}, {"id": 2}]
|
24
|
-
request_response_logs = [
|
25
|
-
AirbyteLogMessage(level=Level.INFO, message="request:{}"),
|
26
|
-
AirbyteLogMessage(level=Level.INFO, message="response{}"),
|
27
|
-
]
|
28
|
-
config = {}
|
29
|
-
|
30
|
-
|
31
|
-
@patch.object(SimpleRetriever, "_read_pages", return_value=iter([]))
|
32
|
-
def test_simple_retriever_full(mock_http_stream):
|
33
|
-
requester = MagicMock()
|
34
|
-
request_params = {"param": "value"}
|
35
|
-
requester.get_request_params.return_value = request_params
|
36
|
-
|
37
|
-
paginator = MagicMock()
|
38
|
-
next_page_token = {"cursor": "cursor_value"}
|
39
|
-
paginator.path.return_value = None
|
40
|
-
paginator.next_page_token.return_value = next_page_token
|
41
|
-
paginator.get_requesyyt_headers.return_value = {}
|
42
|
-
|
43
|
-
record_selector = MagicMock()
|
44
|
-
record_selector.select_records.return_value = records
|
45
|
-
|
46
|
-
cursor = MagicMock(spec=Cursor)
|
47
|
-
stream_slices = [{"date": "2022-01-01"}, {"date": "2022-01-02"}]
|
48
|
-
cursor.stream_slices.return_value = stream_slices
|
49
|
-
|
50
|
-
response = requests.Response()
|
51
|
-
response.status_code = 200
|
52
|
-
|
53
|
-
underlying_state = {"date": "2021-01-01"}
|
54
|
-
cursor.get_stream_state.return_value = underlying_state
|
55
|
-
|
56
|
-
requester.get_authenticator.return_value = NoAuth({})
|
57
|
-
url_base = "https://airbyte.io"
|
58
|
-
requester.get_url_base.return_value = url_base
|
59
|
-
path = "/v1"
|
60
|
-
requester.get_path.return_value = path
|
61
|
-
http_method = HttpMethod.GET
|
62
|
-
requester.get_method.return_value = http_method
|
63
|
-
backoff_time = 60
|
64
|
-
should_retry = ResponseStatus.retry(backoff_time)
|
65
|
-
requester.interpret_response_status.return_value = should_retry
|
66
|
-
request_body_json = {"body": "json"}
|
67
|
-
requester.request_body_json.return_value = request_body_json
|
68
|
-
|
69
|
-
request_body_data = {"body": "data"}
|
70
|
-
requester.get_request_body_data.return_value = request_body_data
|
71
|
-
request_body_json = {"body": "json"}
|
72
|
-
requester.get_request_body_json.return_value = request_body_json
|
73
|
-
request_kwargs = {"kwarg": "value"}
|
74
|
-
requester.request_kwargs.return_value = request_kwargs
|
75
|
-
|
76
|
-
retriever = SimpleRetriever(
|
77
|
-
name="stream_name",
|
78
|
-
primary_key=primary_key,
|
79
|
-
requester=requester,
|
80
|
-
paginator=paginator,
|
81
|
-
record_selector=record_selector,
|
82
|
-
stream_slicer=cursor,
|
83
|
-
cursor=cursor,
|
84
|
-
parameters={},
|
85
|
-
config={},
|
86
|
-
)
|
87
|
-
|
88
|
-
assert retriever.primary_key == primary_key
|
89
|
-
assert retriever.state == underlying_state
|
90
|
-
assert retriever._next_page_token(response) == next_page_token
|
91
|
-
assert retriever._request_params(None, None, None) == {}
|
92
|
-
assert retriever.stream_slices() == stream_slices
|
93
|
-
|
94
|
-
assert retriever._last_response is None
|
95
|
-
assert retriever._records_from_last_response == []
|
96
|
-
assert retriever._parse_response(response, stream_state={}, records_schema={}) == records
|
97
|
-
assert retriever._last_response == response
|
98
|
-
assert retriever._records_from_last_response == records
|
99
|
-
|
100
|
-
[r for r in retriever.read_records(SyncMode.full_refresh)]
|
101
|
-
paginator.reset.assert_called()
|
102
|
-
|
103
|
-
|
104
|
-
@patch.object(SimpleRetriever, "_read_pages", return_value=iter([*request_response_logs, *records]))
|
105
|
-
def test_simple_retriever_with_request_response_logs(mock_http_stream):
|
106
|
-
requester = MagicMock()
|
107
|
-
paginator = MagicMock()
|
108
|
-
record_selector = MagicMock()
|
109
|
-
stream_slicer = DatetimeBasedCursor(
|
110
|
-
start_datetime="",
|
111
|
-
end_datetime="",
|
112
|
-
step="P1D",
|
113
|
-
cursor_field="id",
|
114
|
-
datetime_format="",
|
115
|
-
cursor_granularity="P1D",
|
116
|
-
config={},
|
117
|
-
parameters={},
|
118
|
-
)
|
119
|
-
|
120
|
-
retriever = SimpleRetriever(
|
121
|
-
name="stream_name",
|
122
|
-
primary_key=primary_key,
|
123
|
-
requester=requester,
|
124
|
-
paginator=paginator,
|
125
|
-
record_selector=record_selector,
|
126
|
-
stream_slicer=stream_slicer,
|
127
|
-
parameters={},
|
128
|
-
config={},
|
129
|
-
)
|
130
|
-
|
131
|
-
actual_messages = [r for r in retriever.read_records(SyncMode.full_refresh)]
|
132
|
-
paginator.reset.assert_called()
|
133
|
-
|
134
|
-
assert isinstance(actual_messages[0], AirbyteLogMessage)
|
135
|
-
assert isinstance(actual_messages[1], AirbyteLogMessage)
|
136
|
-
assert actual_messages[2] == records[0]
|
137
|
-
assert actual_messages[3] == records[1]
|
138
|
-
|
139
|
-
|
140
|
-
@patch.object(SimpleRetriever, "_read_pages", return_value=iter([]))
|
141
|
-
def test_simple_retriever_with_request_response_log_last_records(mock_http_stream):
|
142
|
-
requester = MagicMock()
|
143
|
-
paginator = MagicMock()
|
144
|
-
record_selector = MagicMock()
|
145
|
-
record_selector.select_records.return_value = request_response_logs
|
146
|
-
response = requests.Response()
|
147
|
-
response.status_code = 200
|
148
|
-
stream_slicer = DatetimeBasedCursor(
|
149
|
-
start_datetime="",
|
150
|
-
end_datetime="",
|
151
|
-
step="P1D",
|
152
|
-
cursor_field="id",
|
153
|
-
datetime_format="",
|
154
|
-
cursor_granularity="P1D",
|
155
|
-
config={},
|
156
|
-
parameters={},
|
157
|
-
)
|
158
|
-
|
159
|
-
retriever = SimpleRetriever(
|
160
|
-
name="stream_name",
|
161
|
-
primary_key=primary_key,
|
162
|
-
requester=requester,
|
163
|
-
paginator=paginator,
|
164
|
-
record_selector=record_selector,
|
165
|
-
stream_slicer=stream_slicer,
|
166
|
-
parameters={},
|
167
|
-
config={},
|
168
|
-
)
|
169
|
-
|
170
|
-
assert retriever._last_response is None
|
171
|
-
assert retriever._records_from_last_response == []
|
172
|
-
assert retriever._parse_response(response, stream_state={}, records_schema={}) == request_response_logs
|
173
|
-
assert retriever._last_response == response
|
174
|
-
assert retriever._records_from_last_response == request_response_logs
|
175
|
-
|
176
|
-
[r for r in retriever.read_records(SyncMode.full_refresh)]
|
177
|
-
paginator.reset.assert_called()
|
178
|
-
|
179
|
-
|
180
|
-
@pytest.mark.parametrize(
|
181
|
-
"test_name, paginator_mapping, stream_slicer_mapping, expected_mapping",
|
182
|
-
[
|
183
|
-
("test_empty_headers", {}, {}, {}),
|
184
|
-
("test_header_from_pagination_and_slicer", {"offset": 1000}, {"key": "value"}, {"key": "value", "offset": 1000}),
|
185
|
-
("test_header_from_stream_slicer", {}, {"slice": "slice_value"}, {"slice": "slice_value"}),
|
186
|
-
("test_duplicate_header_slicer_paginator", {"k": "v"}, {"k": "slice_value"}, None),
|
187
|
-
],
|
188
|
-
)
|
189
|
-
def test_get_request_options_from_pagination(test_name, paginator_mapping, stream_slicer_mapping, expected_mapping):
|
190
|
-
# This test does not test request headers because they must be strings
|
191
|
-
paginator = MagicMock()
|
192
|
-
paginator.get_request_params.return_value = paginator_mapping
|
193
|
-
paginator.get_request_body_data.return_value = paginator_mapping
|
194
|
-
paginator.get_request_body_json.return_value = paginator_mapping
|
195
|
-
|
196
|
-
stream_slicer = MagicMock()
|
197
|
-
stream_slicer.get_request_params.return_value = stream_slicer_mapping
|
198
|
-
stream_slicer.get_request_body_data.return_value = stream_slicer_mapping
|
199
|
-
stream_slicer.get_request_body_json.return_value = stream_slicer_mapping
|
200
|
-
|
201
|
-
record_selector = MagicMock()
|
202
|
-
retriever = SimpleRetriever(
|
203
|
-
name="stream_name",
|
204
|
-
primary_key=primary_key,
|
205
|
-
requester=MagicMock(),
|
206
|
-
record_selector=record_selector,
|
207
|
-
paginator=paginator,
|
208
|
-
stream_slicer=stream_slicer,
|
209
|
-
parameters={},
|
210
|
-
config={},
|
211
|
-
)
|
212
|
-
|
213
|
-
request_option_type_to_method = {
|
214
|
-
RequestOptionType.request_parameter: retriever._request_params,
|
215
|
-
RequestOptionType.body_data: retriever._request_body_data,
|
216
|
-
RequestOptionType.body_json: retriever._request_body_json,
|
217
|
-
}
|
218
|
-
|
219
|
-
for _, method in request_option_type_to_method.items():
|
220
|
-
if expected_mapping is not None:
|
221
|
-
actual_mapping = method(None, None, None)
|
222
|
-
assert expected_mapping == actual_mapping
|
223
|
-
else:
|
224
|
-
try:
|
225
|
-
method(None, None, None)
|
226
|
-
assert False
|
227
|
-
except ValueError:
|
228
|
-
pass
|
229
|
-
|
230
|
-
|
231
|
-
@pytest.mark.parametrize(
|
232
|
-
"test_name, paginator_mapping, expected_mapping",
|
233
|
-
[
|
234
|
-
("test_only_base_headers", {}, {"key": "value"}),
|
235
|
-
("test_header_from_pagination", {"offset": 1000}, {"key": "value", "offset": "1000"}),
|
236
|
-
("test_duplicate_header", {"key": 1000}, None),
|
237
|
-
],
|
238
|
-
)
|
239
|
-
def test_get_request_headers(test_name, paginator_mapping, expected_mapping):
|
240
|
-
# This test is separate from the other request options because request headers must be strings
|
241
|
-
paginator = MagicMock()
|
242
|
-
paginator.get_request_headers.return_value = paginator_mapping
|
243
|
-
requester = MagicMock(use_cache=False)
|
244
|
-
|
245
|
-
stream_slicer = MagicMock()
|
246
|
-
stream_slicer.get_request_headers.return_value = {"key": "value"}
|
247
|
-
|
248
|
-
record_selector = MagicMock()
|
249
|
-
retriever = SimpleRetriever(
|
250
|
-
name="stream_name",
|
251
|
-
primary_key=primary_key,
|
252
|
-
requester=requester,
|
253
|
-
record_selector=record_selector,
|
254
|
-
stream_slicer=stream_slicer,
|
255
|
-
paginator=paginator,
|
256
|
-
parameters={},
|
257
|
-
config={},
|
258
|
-
)
|
259
|
-
|
260
|
-
request_option_type_to_method = {
|
261
|
-
RequestOptionType.header: retriever._request_headers,
|
262
|
-
}
|
263
|
-
|
264
|
-
for _, method in request_option_type_to_method.items():
|
265
|
-
if expected_mapping:
|
266
|
-
actual_mapping = method(None, None, None)
|
267
|
-
assert expected_mapping == actual_mapping
|
268
|
-
else:
|
269
|
-
try:
|
270
|
-
method(None, None, None)
|
271
|
-
assert False
|
272
|
-
except ValueError:
|
273
|
-
pass
|
274
|
-
|
275
|
-
|
276
|
-
@pytest.mark.parametrize(
|
277
|
-
"test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping",
|
278
|
-
[
|
279
|
-
("test_do_not_ignore_stream_slicer_params_if_ignore_is_true_but_no_next_page_token", {"key_from_pagination": "1000"}, True, None, {"key_from_pagination": "1000"}),
|
280
|
-
("test_do_not_ignore_stream_slicer_params_if_ignore_is_false_and_no_next_page_token", {"key_from_pagination": "1000"}, False, None, {"key_from_pagination": "1000", "key_from_slicer": "value"}),
|
281
|
-
("test_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, True, {"page": 2}, {"key_from_pagination": "1000"}),
|
282
|
-
("test_do_not_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, False, {"page": 2}, {"key_from_pagination": "1000", "key_from_slicer": "value"}),
|
283
|
-
],
|
284
|
-
)
|
285
|
-
def test_ignore_stream_slicer_parameters_on_paginated_requests(test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping):
|
286
|
-
# This test is separate from the other request options because request headers must be strings
|
287
|
-
paginator = MagicMock()
|
288
|
-
paginator.get_request_headers.return_value = paginator_mapping
|
289
|
-
requester = MagicMock(use_cache=False)
|
290
|
-
|
291
|
-
stream_slicer = MagicMock()
|
292
|
-
stream_slicer.get_request_headers.return_value = {"key_from_slicer": "value"}
|
293
|
-
|
294
|
-
record_selector = MagicMock()
|
295
|
-
retriever = SimpleRetriever(
|
296
|
-
name="stream_name",
|
297
|
-
primary_key=primary_key,
|
298
|
-
requester=requester,
|
299
|
-
record_selector=record_selector,
|
300
|
-
stream_slicer=stream_slicer,
|
301
|
-
paginator=paginator,
|
302
|
-
ignore_stream_slicer_parameters_on_paginated_requests=ignore_stream_slicer_parameters_on_paginated_requests,
|
303
|
-
parameters={},
|
304
|
-
config={},
|
305
|
-
)
|
306
|
-
|
307
|
-
request_option_type_to_method = {
|
308
|
-
RequestOptionType.header: retriever._request_headers,
|
309
|
-
}
|
310
|
-
|
311
|
-
for _, method in request_option_type_to_method.items():
|
312
|
-
actual_mapping = method(None, None, next_page_token={"next_page_token": "1000"})
|
313
|
-
assert expected_mapping == actual_mapping
|
314
|
-
|
315
|
-
|
316
|
-
@pytest.mark.parametrize(
|
317
|
-
"test_name, slicer_body_data, paginator_body_data, expected_body_data",
|
318
|
-
[
|
319
|
-
("test_only_slicer_mapping", {"key": "value"}, {}, {"key": "value"}),
|
320
|
-
("test_only_slicer_string", "key=value", {}, "key=value"),
|
321
|
-
("test_slicer_mapping_and_paginator_no_duplicate", {"key": "value"}, {"offset": 1000}, {"key": "value", "offset": 1000}),
|
322
|
-
("test_slicer_mapping_and_paginator_with_duplicate", {"key": "value"}, {"key": 1000}, None),
|
323
|
-
("test_slicer_string_and_paginator", "key=value", {"offset": 1000}, None),
|
324
|
-
],
|
325
|
-
)
|
326
|
-
def test_request_body_data(test_name, slicer_body_data, paginator_body_data, expected_body_data):
|
327
|
-
paginator = MagicMock()
|
328
|
-
paginator.get_request_body_data.return_value = paginator_body_data
|
329
|
-
requester = MagicMock(use_cache=False)
|
330
|
-
|
331
|
-
stream_slicer = MagicMock()
|
332
|
-
stream_slicer.get_request_body_data.return_value = slicer_body_data
|
333
|
-
|
334
|
-
record_selector = MagicMock()
|
335
|
-
retriever = SimpleRetriever(
|
336
|
-
name="stream_name",
|
337
|
-
primary_key=primary_key,
|
338
|
-
requester=requester,
|
339
|
-
record_selector=record_selector,
|
340
|
-
paginator=paginator,
|
341
|
-
stream_slicer=stream_slicer,
|
342
|
-
parameters={},
|
343
|
-
config={},
|
344
|
-
)
|
345
|
-
|
346
|
-
if expected_body_data:
|
347
|
-
actual_body_data = retriever._request_body_data(None, None, None)
|
348
|
-
assert expected_body_data == actual_body_data
|
349
|
-
else:
|
350
|
-
try:
|
351
|
-
retriever._request_body_data(None, None, None)
|
352
|
-
assert False
|
353
|
-
except ValueError:
|
354
|
-
pass
|
355
|
-
|
356
|
-
|
357
|
-
@pytest.mark.parametrize(
|
358
|
-
"test_name, requester_path, paginator_path, expected_path",
|
359
|
-
[
|
360
|
-
("test_path_from_requester", "/v1/path", None, None),
|
361
|
-
("test_path_from_paginator", "/v1/path/", "/v2/paginator", "/v2/paginator"),
|
362
|
-
],
|
363
|
-
)
|
364
|
-
def test_path(test_name, requester_path, paginator_path, expected_path):
|
365
|
-
paginator = MagicMock()
|
366
|
-
paginator.path.return_value = paginator_path
|
367
|
-
requester = MagicMock(use_cache=False)
|
368
|
-
|
369
|
-
requester.get_path.return_value = requester_path
|
370
|
-
|
371
|
-
record_selector = MagicMock()
|
372
|
-
retriever = SimpleRetriever(
|
373
|
-
name="stream_name",
|
374
|
-
primary_key=primary_key,
|
375
|
-
requester=requester,
|
376
|
-
record_selector=record_selector,
|
377
|
-
paginator=paginator,
|
378
|
-
parameters={},
|
379
|
-
config={},
|
380
|
-
)
|
381
|
-
|
382
|
-
actual_path = retriever._paginator_path()
|
383
|
-
assert expected_path == actual_path
|
384
|
-
|
385
|
-
|
386
|
-
def test_limit_stream_slices():
|
387
|
-
maximum_number_of_slices = 4
|
388
|
-
stream_slicer = MagicMock()
|
389
|
-
stream_slicer.stream_slices.return_value = _generate_slices(maximum_number_of_slices * 2)
|
390
|
-
retriever = SimpleRetrieverTestReadDecorator(
|
391
|
-
name="stream_name",
|
392
|
-
primary_key=primary_key,
|
393
|
-
requester=MagicMock(),
|
394
|
-
paginator=MagicMock(),
|
395
|
-
record_selector=MagicMock(),
|
396
|
-
stream_slicer=stream_slicer,
|
397
|
-
maximum_number_of_slices=maximum_number_of_slices,
|
398
|
-
parameters={},
|
399
|
-
config={},
|
400
|
-
)
|
401
|
-
|
402
|
-
truncated_slices = list(retriever.stream_slices())
|
403
|
-
|
404
|
-
assert truncated_slices == _generate_slices(maximum_number_of_slices)
|
405
|
-
|
406
|
-
|
407
|
-
@pytest.mark.parametrize(
|
408
|
-
"test_name, first_greater_than_second",
|
409
|
-
[
|
410
|
-
("test_first_greater_than_second", True),
|
411
|
-
("test_second_greater_than_first", False),
|
412
|
-
],
|
413
|
-
)
|
414
|
-
def test_when_read_records_then_cursor_close_slice_with_greater_record(test_name, first_greater_than_second):
|
415
|
-
first_record = Record({"first": 1}, {})
|
416
|
-
second_record = Record({"second": 2}, {})
|
417
|
-
records = [first_record, second_record]
|
418
|
-
record_selector = MagicMock()
|
419
|
-
record_selector.select_records.return_value = records
|
420
|
-
cursor = MagicMock(spec=Cursor)
|
421
|
-
cursor.is_greater_than_or_equal.return_value = first_greater_than_second
|
422
|
-
paginator = MagicMock()
|
423
|
-
paginator.get_request_headers.return_value = {}
|
424
|
-
|
425
|
-
retriever = SimpleRetriever(
|
426
|
-
name="stream_name",
|
427
|
-
primary_key=primary_key,
|
428
|
-
requester=MagicMock(),
|
429
|
-
paginator=paginator,
|
430
|
-
record_selector=record_selector,
|
431
|
-
stream_slicer=cursor,
|
432
|
-
cursor=cursor,
|
433
|
-
parameters={},
|
434
|
-
config={},
|
435
|
-
)
|
436
|
-
stream_slice = {"repository": "airbyte"}
|
437
|
-
|
438
|
-
def retriever_read_pages(_, __, ___):
|
439
|
-
return retriever._parse_records(response=MagicMock(), stream_state={}, stream_slice=stream_slice, records_schema={})
|
440
|
-
|
441
|
-
with patch.object(
|
442
|
-
SimpleRetriever,
|
443
|
-
"_read_pages",
|
444
|
-
return_value=iter([first_record, second_record]),
|
445
|
-
side_effect=retriever_read_pages,
|
446
|
-
):
|
447
|
-
list(retriever.read_records(stream_slice=stream_slice, records_schema={}))
|
448
|
-
cursor.close_slice.assert_called_once_with(stream_slice, first_record if first_greater_than_second else second_record)
|
449
|
-
|
450
|
-
|
451
|
-
def test_given_stream_data_is_not_record_when_read_records_then_update_slice_with_optional_record():
|
452
|
-
stream_data = [AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="a log message"))]
|
453
|
-
record_selector = MagicMock()
|
454
|
-
record_selector.select_records.return_value = []
|
455
|
-
cursor = MagicMock(spec=Cursor)
|
456
|
-
|
457
|
-
retriever = SimpleRetriever(
|
458
|
-
name="stream_name",
|
459
|
-
primary_key=primary_key,
|
460
|
-
requester=MagicMock(),
|
461
|
-
paginator=Mock(),
|
462
|
-
record_selector=record_selector,
|
463
|
-
stream_slicer=cursor,
|
464
|
-
cursor=cursor,
|
465
|
-
parameters={},
|
466
|
-
config={},
|
467
|
-
)
|
468
|
-
stream_slice = {"repository": "airbyte"}
|
469
|
-
|
470
|
-
def retriever_read_pages(_, __, ___):
|
471
|
-
return retriever._parse_records(response=MagicMock(), stream_state={}, stream_slice=stream_slice, records_schema={})
|
472
|
-
|
473
|
-
with patch.object(
|
474
|
-
SimpleRetriever,
|
475
|
-
"_read_pages",
|
476
|
-
return_value=iter(stream_data),
|
477
|
-
side_effect=retriever_read_pages,
|
478
|
-
):
|
479
|
-
list(retriever.read_records(stream_slice=stream_slice, records_schema={}))
|
480
|
-
cursor.observe.assert_not_called()
|
481
|
-
cursor.close_slice.assert_called_once_with(stream_slice, None)
|
482
|
-
|
483
|
-
|
484
|
-
def _generate_slices(number_of_slices):
|
485
|
-
return [{"date": f"2022-01-0{day + 1}"} for day in range(number_of_slices)]
|
486
|
-
|
487
|
-
|
488
|
-
@patch.object(SimpleRetriever, "_read_pages", return_value=iter([]))
|
489
|
-
def test_given_state_selector_when_read_records_use_stream_state(http_stream_read_pages, mocker):
|
490
|
-
requester = MagicMock()
|
491
|
-
paginator = MagicMock()
|
492
|
-
record_selector = MagicMock()
|
493
|
-
cursor = MagicMock(spec=Cursor)
|
494
|
-
cursor.select_state = MagicMock(return_value=A_SLICE_STATE)
|
495
|
-
cursor.get_stream_state = MagicMock(return_value=A_STREAM_STATE)
|
496
|
-
|
497
|
-
retriever = SimpleRetriever(
|
498
|
-
name="stream_name",
|
499
|
-
primary_key=primary_key,
|
500
|
-
requester=requester,
|
501
|
-
paginator=paginator,
|
502
|
-
record_selector=record_selector,
|
503
|
-
stream_slicer=cursor,
|
504
|
-
cursor=cursor,
|
505
|
-
parameters={},
|
506
|
-
config={},
|
507
|
-
)
|
508
|
-
|
509
|
-
list(retriever.read_records(stream_slice=A_STREAM_SLICE, records_schema={}))
|
510
|
-
|
511
|
-
http_stream_read_pages.assert_called_once_with(mocker.ANY, A_STREAM_STATE, A_STREAM_SLICE)
|
512
|
-
|
513
|
-
|
514
|
-
def test_emit_log_request_response_messages(mocker):
|
515
|
-
record_selector = MagicMock()
|
516
|
-
record_selector.select_records.return_value = records
|
517
|
-
|
518
|
-
request = requests.PreparedRequest()
|
519
|
-
request.headers = {"header": "value"}
|
520
|
-
request.url = "http://byrde.enterprises.com/casinos"
|
521
|
-
|
522
|
-
response = requests.Response()
|
523
|
-
response.request = request
|
524
|
-
response.status_code = 200
|
525
|
-
|
526
|
-
format_http_message_mock = mocker.patch("airbyte_cdk.sources.declarative.retrievers.simple_retriever.format_http_message")
|
527
|
-
requester = MagicMock()
|
528
|
-
retriever = SimpleRetrieverTestReadDecorator(
|
529
|
-
name="stream_name",
|
530
|
-
primary_key=primary_key,
|
531
|
-
requester=requester,
|
532
|
-
paginator=MagicMock(),
|
533
|
-
record_selector=record_selector,
|
534
|
-
stream_slicer=SinglePartitionRouter(parameters={}),
|
535
|
-
parameters={},
|
536
|
-
config={},
|
537
|
-
)
|
538
|
-
|
539
|
-
retriever._fetch_next_page(stream_state={}, stream_slice={})
|
540
|
-
|
541
|
-
assert requester.send_request.call_args_list[0][1]["log_formatter"] is not None
|
542
|
-
assert requester.send_request.call_args_list[0][1]["log_formatter"](response) == format_http_message_mock.return_value
|
@@ -1,32 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from unittest.mock import MagicMock
|
6
|
-
|
7
|
-
import pytest
|
8
|
-
from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader
|
9
|
-
|
10
|
-
|
11
|
-
@pytest.mark.parametrize(
|
12
|
-
"found_schema, found_error, expected_schema",
|
13
|
-
[
|
14
|
-
pytest.param(
|
15
|
-
{"type": "object", "properties": {}}, None, {"type": "object", "properties": {}}, id="test_has_schema_in_default_location"
|
16
|
-
),
|
17
|
-
pytest.param(None, FileNotFoundError, {}, id="test_schema_file_does_not_exist"),
|
18
|
-
],
|
19
|
-
)
|
20
|
-
def test_get_json_schema(found_schema, found_error, expected_schema):
|
21
|
-
default_schema_loader = DefaultSchemaLoader({}, {})
|
22
|
-
|
23
|
-
json_file_schema_loader = MagicMock()
|
24
|
-
if found_schema:
|
25
|
-
json_file_schema_loader.get_json_schema.return_value = {"type": "object", "properties": {}}
|
26
|
-
if found_error:
|
27
|
-
json_file_schema_loader.get_json_schema.side_effect = found_error
|
28
|
-
|
29
|
-
default_schema_loader.default_loader = json_file_schema_loader
|
30
|
-
|
31
|
-
actual_schema = default_schema_loader.get_json_schema()
|
32
|
-
assert actual_schema == expected_schema
|
@@ -1,19 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
import pytest
|
6
|
-
from airbyte_cdk.sources.declarative.schema import InlineSchemaLoader
|
7
|
-
|
8
|
-
|
9
|
-
@pytest.mark.parametrize(
|
10
|
-
"test_name, input_schema, expected_schema",
|
11
|
-
[
|
12
|
-
("schema", {"k": "string"}, {"k": "string"}),
|
13
|
-
("empty_schema", {}, {}),
|
14
|
-
],
|
15
|
-
)
|
16
|
-
def test_static_schema_loads(test_name, input_schema, expected_schema):
|
17
|
-
schema_loader = InlineSchemaLoader(input_schema, {})
|
18
|
-
|
19
|
-
assert schema_loader.get_json_schema() == expected_schema
|
@@ -1,26 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
import pytest
|
6
|
-
from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader
|
7
|
-
|
8
|
-
|
9
|
-
@pytest.mark.parametrize(
|
10
|
-
"test_name, input_path, expected_resource, expected_path",
|
11
|
-
[
|
12
|
-
("path_prefixed_with_dot", "./source_example/schemas/lists.json", "source_example", "schemas/lists.json"),
|
13
|
-
("path_prefixed_with_slash", "/source_example/schemas/lists.json", "source_example", "schemas/lists.json"),
|
14
|
-
("path_starting_with_source", "source_example/schemas/lists.json", "source_example", "schemas/lists.json"),
|
15
|
-
("path_starting_missing_source", "schemas/lists.json", "schemas", "lists.json"),
|
16
|
-
("path_with_file_only", "lists.json", "", "lists.json"),
|
17
|
-
("empty_path_does_not_crash", "", "", ""),
|
18
|
-
("empty_path_with_slash_does_not_crash", "/", "", ""),
|
19
|
-
],
|
20
|
-
)
|
21
|
-
def test_extract_resource_and_schema_path(test_name, input_path, expected_resource, expected_path):
|
22
|
-
json_schema = JsonFileSchemaLoader({}, {}, input_path)
|
23
|
-
actual_resource, actual_path = json_schema.extract_resource_and_schema_path(input_path)
|
24
|
-
|
25
|
-
assert actual_resource == expected_resource
|
26
|
-
assert actual_path == expected_path
|