airbyte-cdk 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/__init__.py +358 -0
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +236 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +104 -0
- airbyte_cdk/connector.py +123 -0
- airbyte_cdk/connector_builder/README.md +53 -0
- airbyte_cdk/connector_builder/__init__.py +3 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +121 -0
- airbyte_cdk/connector_builder/main.py +107 -0
- airbyte_cdk/connector_builder/models.py +73 -0
- airbyte_cdk/connector_builder/test_reader/__init__.py +7 -0
- airbyte_cdk/connector_builder/test_reader/helpers.py +689 -0
- airbyte_cdk/connector_builder/test_reader/message_grouper.py +173 -0
- airbyte_cdk/connector_builder/test_reader/reader.py +441 -0
- airbyte_cdk/connector_builder/test_reader/types.py +83 -0
- airbyte_cdk/destinations/__init__.py +8 -0
- airbyte_cdk/destinations/destination.py +154 -0
- airbyte_cdk/destinations/vector_db_based/README.md +37 -0
- airbyte_cdk/destinations/vector_db_based/__init__.py +38 -0
- airbyte_cdk/destinations/vector_db_based/config.py +298 -0
- airbyte_cdk/destinations/vector_db_based/document_processor.py +223 -0
- airbyte_cdk/destinations/vector_db_based/embedder.py +303 -0
- airbyte_cdk/destinations/vector_db_based/indexer.py +78 -0
- airbyte_cdk/destinations/vector_db_based/test_utils.py +63 -0
- airbyte_cdk/destinations/vector_db_based/utils.py +35 -0
- airbyte_cdk/destinations/vector_db_based/writer.py +104 -0
- airbyte_cdk/entrypoint.py +414 -0
- airbyte_cdk/exception_handler.py +56 -0
- airbyte_cdk/logger.py +109 -0
- airbyte_cdk/models/__init__.py +72 -0
- airbyte_cdk/models/airbyte_protocol.py +88 -0
- airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
- airbyte_cdk/models/well_known_types.py +5 -0
- airbyte_cdk/py.typed +0 -0
- airbyte_cdk/sources/__init__.py +26 -0
- airbyte_cdk/sources/abstract_source.py +326 -0
- airbyte_cdk/sources/concurrent_source/__init__.py +8 -0
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +255 -0
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +165 -0
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +147 -0
- airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py +24 -0
- airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +115 -0
- airbyte_cdk/sources/config.py +27 -0
- airbyte_cdk/sources/connector_state_manager.py +161 -0
- airbyte_cdk/sources/declarative/__init__.py +3 -0
- airbyte_cdk/sources/declarative/async_job/__init__.py +0 -0
- airbyte_cdk/sources/declarative/async_job/job.py +52 -0
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +525 -0
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +79 -0
- airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
- airbyte_cdk/sources/declarative/async_job/status.py +24 -0
- airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
- airbyte_cdk/sources/declarative/auth/__init__.py +8 -0
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +42 -0
- airbyte_cdk/sources/declarative/auth/jwt.py +197 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +293 -0
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +45 -0
- airbyte_cdk/sources/declarative/auth/token.py +267 -0
- airbyte_cdk/sources/declarative/auth/token_provider.py +82 -0
- airbyte_cdk/sources/declarative/checks/__init__.py +24 -0
- airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +61 -0
- airbyte_cdk/sources/declarative/checks/check_stream.py +56 -0
- airbyte_cdk/sources/declarative/checks/connection_checker.py +35 -0
- airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +526 -0
- airbyte_cdk/sources/declarative/datetime/__init__.py +3 -0
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +65 -0
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +118 -0
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +3975 -0
- airbyte_cdk/sources/declarative/declarative_source.py +36 -0
- airbyte_cdk/sources/declarative/declarative_stream.py +241 -0
- airbyte_cdk/sources/declarative/decoders/__init__.py +33 -0
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +218 -0
- airbyte_cdk/sources/declarative/decoders/decoder.py +32 -0
- airbyte_cdk/sources/declarative/decoders/decoder_parser.py +30 -0
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +65 -0
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
- airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py +56 -0
- airbyte_cdk/sources/declarative/exceptions.py +9 -0
- airbyte_cdk/sources/declarative/extractors/__init__.py +21 -0
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +86 -0
- airbyte_cdk/sources/declarative/extractors/http_selector.py +37 -0
- airbyte_cdk/sources/declarative/extractors/record_extractor.py +27 -0
- airbyte_cdk/sources/declarative/extractors/record_filter.py +91 -0
- airbyte_cdk/sources/declarative/extractors/record_selector.py +170 -0
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +176 -0
- airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
- airbyte_cdk/sources/declarative/incremental/__init__.py +37 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +497 -0
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +459 -0
- airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +357 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +380 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
- airbyte_cdk/sources/declarative/interpolation/__init__.py +9 -0
- airbyte_cdk/sources/declarative/interpolation/filters.py +139 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +66 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +56 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +52 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +79 -0
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +34 -0
- airbyte_cdk/sources/declarative/interpolation/jinja.py +161 -0
- airbyte_cdk/sources/declarative/interpolation/macros.py +191 -0
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +421 -0
- airbyte_cdk/sources/declarative/migrations/__init__.py +0 -0
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
- airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
- airbyte_cdk/sources/declarative/models/__init__.py +2 -0
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2503 -0
- airbyte_cdk/sources/declarative/parsers/__init__.py +3 -0
- airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +157 -0
- airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +21 -0
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +172 -0
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +213 -0
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +3407 -0
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +29 -0
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +121 -0
- airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
- airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +63 -0
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +437 -0
- airbyte_cdk/sources/declarative/requesters/README.md +56 -0
- airbyte_cdk/sources/declarative/requesters/__init__.py +9 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +25 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +23 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +45 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +45 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +41 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +70 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +77 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +17 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +101 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +147 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +17 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +179 -0
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +350 -0
- airbyte_cdk/sources/declarative/requesters/http_requester.py +433 -0
- airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +21 -0
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +327 -0
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +76 -0
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +65 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +25 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +98 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +102 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +71 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +48 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +66 -0
- airbyte_cdk/sources/declarative/requesters/request_option.py +117 -0
- airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +23 -0
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +92 -0
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +59 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +68 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +119 -0
- airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +79 -0
- airbyte_cdk/sources/declarative/requesters/request_path.py +15 -0
- airbyte_cdk/sources/declarative/requesters/requester.py +144 -0
- airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
- airbyte_cdk/sources/declarative/retrievers/__init__.py +19 -0
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +124 -0
- airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
- airbyte_cdk/sources/declarative/retrievers/retriever.py +54 -0
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +702 -0
- airbyte_cdk/sources/declarative/schema/__init__.py +25 -0
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +47 -0
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +285 -0
- airbyte_cdk/sources/declarative/schema/inline_schema_loader.py +19 -0
- airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +92 -0
- airbyte_cdk/sources/declarative/schema/schema_loader.py +17 -0
- airbyte_cdk/sources/declarative/spec/__init__.py +7 -0
- airbyte_cdk/sources/declarative/spec/spec.py +48 -0
- airbyte_cdk/sources/declarative/stream_slicers/__init__.py +7 -0
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +93 -0
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +25 -0
- airbyte_cdk/sources/declarative/transformations/__init__.py +17 -0
- airbyte_cdk/sources/declarative/transformations/add_fields.py +146 -0
- airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py +61 -0
- airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
- airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +75 -0
- airbyte_cdk/sources/declarative/transformations/transformation.py +37 -0
- airbyte_cdk/sources/declarative/types.py +25 -0
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +67 -0
- airbyte_cdk/sources/file_based/README.md +152 -0
- airbyte_cdk/sources/file_based/__init__.py +24 -0
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +11 -0
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +73 -0
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +149 -0
- airbyte_cdk/sources/file_based/config/__init__.py +0 -0
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +153 -0
- airbyte_cdk/sources/file_based/config/avro_format.py +25 -0
- airbyte_cdk/sources/file_based/config/csv_format.py +210 -0
- airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +99 -0
- airbyte_cdk/sources/file_based/config/jsonl_format.py +18 -0
- airbyte_cdk/sources/file_based/config/parquet_format.py +25 -0
- airbyte_cdk/sources/file_based/config/unstructured_format.py +102 -0
- airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py +81 -0
- airbyte_cdk/sources/file_based/discovery_policy/__init__.py +8 -0
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +21 -0
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +33 -0
- airbyte_cdk/sources/file_based/exceptions.py +159 -0
- airbyte_cdk/sources/file_based/file_based_source.py +466 -0
- airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py +123 -0
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +209 -0
- airbyte_cdk/sources/file_based/file_record_data.py +22 -0
- airbyte_cdk/sources/file_based/file_types/__init__.py +37 -0
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +233 -0
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +527 -0
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +30 -0
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +86 -0
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +145 -0
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +275 -0
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +480 -0
- airbyte_cdk/sources/file_based/remote_file.py +18 -0
- airbyte_cdk/sources/file_based/schema_helpers.py +281 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +17 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +20 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +52 -0
- airbyte_cdk/sources/file_based/stream/__init__.py +13 -0
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +197 -0
- airbyte_cdk/sources/file_based/stream/concurrent/__init__.py +0 -0
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +343 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +9 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +59 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +313 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +83 -0
- airbyte_cdk/sources/file_based/stream/cursor/__init__.py +4 -0
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +66 -0
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +149 -0
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +396 -0
- airbyte_cdk/sources/file_based/stream/identities_stream.py +49 -0
- airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +92 -0
- airbyte_cdk/sources/file_based/types.py +10 -0
- airbyte_cdk/sources/http_config.py +10 -0
- airbyte_cdk/sources/http_logger.py +55 -0
- airbyte_cdk/sources/message/__init__.py +19 -0
- airbyte_cdk/sources/message/repository.py +137 -0
- airbyte_cdk/sources/source.py +95 -0
- airbyte_cdk/sources/specs/transfer_modes.py +26 -0
- airbyte_cdk/sources/streams/__init__.py +8 -0
- airbyte_cdk/sources/streams/availability_strategy.py +84 -0
- airbyte_cdk/sources/streams/call_rate.py +704 -0
- airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
- airbyte_cdk/sources/streams/checkpoint/cursor.py +77 -0
- airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
- airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
- airbyte_cdk/sources/streams/concurrent/README.md +7 -0
- airbyte_cdk/sources/streams/concurrent/__init__.py +3 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +96 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py +37 -0
- airbyte_cdk/sources/streams/concurrent/adapters.py +397 -0
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +94 -0
- airbyte_cdk/sources/streams/concurrent/clamping.py +99 -0
- airbyte_cdk/sources/streams/concurrent/cursor.py +481 -0
- airbyte_cdk/sources/streams/concurrent/cursor_types.py +32 -0
- airbyte_cdk/sources/streams/concurrent/default_stream.py +102 -0
- airbyte_cdk/sources/streams/concurrent/exceptions.py +18 -0
- airbyte_cdk/sources/streams/concurrent/helpers.py +42 -0
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +64 -0
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +45 -0
- airbyte_cdk/sources/streams/concurrent/partitions/__init__.py +3 -0
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +48 -0
- airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py +18 -0
- airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +38 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py +0 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +182 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +223 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
- airbyte_cdk/sources/streams/core.py +703 -0
- airbyte_cdk/sources/streams/http/__init__.py +10 -0
- airbyte_cdk/sources/streams/http/availability_strategy.py +54 -0
- airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
- airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
- airbyte_cdk/sources/streams/http/exceptions.py +61 -0
- airbyte_cdk/sources/streams/http/http.py +673 -0
- airbyte_cdk/sources/streams/http/http_client.py +531 -0
- airbyte_cdk/sources/streams/http/rate_limiting.py +158 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py +14 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +479 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +34 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +436 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +83 -0
- airbyte_cdk/sources/streams/permissions/identities_stream.py +75 -0
- airbyte_cdk/sources/streams/utils/__init__.py +3 -0
- airbyte_cdk/sources/types.py +169 -0
- airbyte_cdk/sources/utils/__init__.py +7 -0
- airbyte_cdk/sources/utils/casing.py +12 -0
- airbyte_cdk/sources/utils/files_directory.py +15 -0
- airbyte_cdk/sources/utils/record_helper.py +53 -0
- airbyte_cdk/sources/utils/schema_helpers.py +230 -0
- airbyte_cdk/sources/utils/slice_logger.py +57 -0
- airbyte_cdk/sources/utils/transform.py +277 -0
- airbyte_cdk/sources/utils/types.py +7 -0
- airbyte_cdk/sql/__init__.py +0 -0
- airbyte_cdk/sql/_util/__init__.py +0 -0
- airbyte_cdk/sql/_util/hashing.py +34 -0
- airbyte_cdk/sql/_util/name_normalizers.py +92 -0
- airbyte_cdk/sql/constants.py +32 -0
- airbyte_cdk/sql/exceptions.py +235 -0
- airbyte_cdk/sql/secrets.py +123 -0
- airbyte_cdk/sql/shared/__init__.py +15 -0
- airbyte_cdk/sql/shared/catalog_providers.py +145 -0
- airbyte_cdk/sql/shared/sql_processor.py +786 -0
- airbyte_cdk/sql/types.py +160 -0
- airbyte_cdk/test/__init__.py +7 -0
- airbyte_cdk/test/catalog_builder.py +81 -0
- airbyte_cdk/test/entrypoint_wrapper.py +250 -0
- airbyte_cdk/test/mock_http/__init__.py +6 -0
- airbyte_cdk/test/mock_http/matcher.py +41 -0
- airbyte_cdk/test/mock_http/mocker.py +185 -0
- airbyte_cdk/test/mock_http/request.py +103 -0
- airbyte_cdk/test/mock_http/response.py +28 -0
- airbyte_cdk/test/mock_http/response_builder.py +237 -0
- airbyte_cdk/test/state_builder.py +33 -0
- airbyte_cdk/test/utils/__init__.py +1 -0
- airbyte_cdk/test/utils/data.py +24 -0
- airbyte_cdk/test/utils/http_mocking.py +16 -0
- airbyte_cdk/test/utils/manifest_only_fixtures.py +59 -0
- airbyte_cdk/test/utils/reading.py +26 -0
- airbyte_cdk/utils/__init__.py +10 -0
- airbyte_cdk/utils/airbyte_secrets_utils.py +80 -0
- airbyte_cdk/utils/analytics_message.py +25 -0
- airbyte_cdk/utils/constants.py +5 -0
- airbyte_cdk/utils/datetime_format_inferrer.py +94 -0
- airbyte_cdk/utils/datetime_helpers.py +499 -0
- airbyte_cdk/utils/event_timing.py +85 -0
- airbyte_cdk/utils/is_cloud_environment.py +18 -0
- airbyte_cdk/utils/mapping_helpers.py +162 -0
- airbyte_cdk/utils/message_utils.py +26 -0
- airbyte_cdk/utils/oneof_option_config.py +33 -0
- airbyte_cdk/utils/print_buffer.py +75 -0
- airbyte_cdk/utils/schema_inferrer.py +270 -0
- airbyte_cdk/utils/slice_hasher.py +37 -0
- airbyte_cdk/utils/spec_schema_transformations.py +26 -0
- airbyte_cdk/utils/stream_status_utils.py +43 -0
- airbyte_cdk/utils/traced_exception.py +145 -0
- airbyte_cdk-0.0.0.dev0.dist-info/LICENSE.txt +19 -0
- airbyte_cdk-0.0.0.dev0.dist-info/LICENSE_SHORT +1 -0
- airbyte_cdk-0.0.0.dev0.dist-info/METADATA +111 -0
- airbyte_cdk-0.0.0.dev0.dist-info/RECORD +368 -0
- airbyte_cdk-0.0.0.dev0.dist-info/WHEEL +4 -0
- airbyte_cdk-0.0.0.dev0.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
2
|
+
|
|
3
|
+
import contextlib
|
|
4
|
+
import functools
|
|
5
|
+
from collections import defaultdict
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from types import TracebackType
|
|
8
|
+
from typing import Callable, Dict, Iterable, List, Optional, Union
|
|
9
|
+
|
|
10
|
+
import requests_mock
|
|
11
|
+
|
|
12
|
+
from airbyte_cdk.test.mock_http.matcher import HttpRequestMatcher
|
|
13
|
+
from airbyte_cdk.test.mock_http.request import HttpRequest
|
|
14
|
+
from airbyte_cdk.test.mock_http.response import HttpResponse
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class SupportedHttpMethods(str, Enum):
|
|
18
|
+
GET = "get"
|
|
19
|
+
PATCH = "patch"
|
|
20
|
+
POST = "post"
|
|
21
|
+
PUT = "put"
|
|
22
|
+
DELETE = "delete"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class HttpMocker(contextlib.ContextDecorator):
|
|
26
|
+
"""
|
|
27
|
+
WARNING 1: This implementation only works if the lib used to perform HTTP requests is `requests`.
|
|
28
|
+
|
|
29
|
+
WARNING 2: Given multiple requests that are not mutually exclusive, the request will match the first one. This can happen in scenarios
|
|
30
|
+
where the same request is added twice (in which case there will always be an exception because we will never match the second
|
|
31
|
+
request) or in a case like this:
|
|
32
|
+
```
|
|
33
|
+
http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1", "more_granular": "2"}), <...>)
|
|
34
|
+
http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1"}), <...>)
|
|
35
|
+
requests.get(_A_URL, headers={"less_granular": "1", "more_granular": "2"})
|
|
36
|
+
```
|
|
37
|
+
In the example above, the matcher would match the second mock as requests_mock iterate over the matcher in reverse order (see
|
|
38
|
+
https://github.com/jamielennox/requests-mock/blob/c06f124a33f56e9f03840518e19669ba41b93202/requests_mock/adapter.py#L246) even
|
|
39
|
+
though the request sent is a better match for the first `http_mocker.get`.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(self) -> None:
|
|
43
|
+
self._mocker = requests_mock.Mocker()
|
|
44
|
+
self._matchers: Dict[SupportedHttpMethods, List[HttpRequestMatcher]] = defaultdict(list)
|
|
45
|
+
|
|
46
|
+
def __enter__(self) -> "HttpMocker":
|
|
47
|
+
self._mocker.__enter__()
|
|
48
|
+
return self
|
|
49
|
+
|
|
50
|
+
def __exit__(
|
|
51
|
+
self,
|
|
52
|
+
exc_type: Optional[BaseException],
|
|
53
|
+
exc_val: Optional[BaseException],
|
|
54
|
+
exc_tb: Optional[TracebackType],
|
|
55
|
+
) -> None:
|
|
56
|
+
self._mocker.__exit__(exc_type, exc_val, exc_tb)
|
|
57
|
+
|
|
58
|
+
def _validate_all_matchers_called(self) -> None:
|
|
59
|
+
for matcher in self._get_matchers():
|
|
60
|
+
if not matcher.has_expected_match_count():
|
|
61
|
+
raise ValueError(f"Invalid number of matches for `{matcher}`")
|
|
62
|
+
|
|
63
|
+
def _mock_request_method(
|
|
64
|
+
self,
|
|
65
|
+
method: SupportedHttpMethods,
|
|
66
|
+
request: HttpRequest,
|
|
67
|
+
responses: Union[HttpResponse, List[HttpResponse]],
|
|
68
|
+
) -> None:
|
|
69
|
+
if isinstance(responses, HttpResponse):
|
|
70
|
+
responses = [responses]
|
|
71
|
+
|
|
72
|
+
matcher = HttpRequestMatcher(request, len(responses))
|
|
73
|
+
if matcher in self._matchers[method]:
|
|
74
|
+
raise ValueError(f"Request {matcher.request} already mocked")
|
|
75
|
+
self._matchers[method].append(matcher)
|
|
76
|
+
|
|
77
|
+
getattr(self._mocker, method)(
|
|
78
|
+
requests_mock.ANY,
|
|
79
|
+
additional_matcher=self._matches_wrapper(matcher),
|
|
80
|
+
response_list=[
|
|
81
|
+
{
|
|
82
|
+
self._get_body_field(response): response.body,
|
|
83
|
+
"status_code": response.status_code,
|
|
84
|
+
"headers": response.headers,
|
|
85
|
+
}
|
|
86
|
+
for response in responses
|
|
87
|
+
],
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
@staticmethod
|
|
91
|
+
def _get_body_field(response: HttpResponse) -> str:
|
|
92
|
+
return "text" if isinstance(response.body, str) else "content"
|
|
93
|
+
|
|
94
|
+
def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
|
|
95
|
+
self._mock_request_method(SupportedHttpMethods.GET, request, responses)
|
|
96
|
+
|
|
97
|
+
def patch(
|
|
98
|
+
self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
|
|
99
|
+
) -> None:
|
|
100
|
+
self._mock_request_method(SupportedHttpMethods.PATCH, request, responses)
|
|
101
|
+
|
|
102
|
+
def post(
|
|
103
|
+
self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
|
|
104
|
+
) -> None:
|
|
105
|
+
self._mock_request_method(SupportedHttpMethods.POST, request, responses)
|
|
106
|
+
|
|
107
|
+
def put(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None:
|
|
108
|
+
self._mock_request_method(SupportedHttpMethods.PUT, request, responses)
|
|
109
|
+
|
|
110
|
+
def delete(
|
|
111
|
+
self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]
|
|
112
|
+
) -> None:
|
|
113
|
+
self._mock_request_method(SupportedHttpMethods.DELETE, request, responses)
|
|
114
|
+
|
|
115
|
+
@staticmethod
|
|
116
|
+
def _matches_wrapper(
|
|
117
|
+
matcher: HttpRequestMatcher,
|
|
118
|
+
) -> Callable[[requests_mock.request._RequestObjectProxy], bool]:
|
|
119
|
+
def matches(requests_mock_request: requests_mock.request._RequestObjectProxy) -> bool:
|
|
120
|
+
# query_params are provided as part of `requests_mock_request.url`
|
|
121
|
+
http_request = HttpRequest(
|
|
122
|
+
requests_mock_request.url,
|
|
123
|
+
query_params={},
|
|
124
|
+
headers=requests_mock_request.headers,
|
|
125
|
+
body=requests_mock_request.body,
|
|
126
|
+
)
|
|
127
|
+
return matcher.matches(http_request)
|
|
128
|
+
|
|
129
|
+
return matches
|
|
130
|
+
|
|
131
|
+
def assert_number_of_calls(self, request: HttpRequest, number_of_calls: int) -> None:
|
|
132
|
+
corresponding_matchers = list(
|
|
133
|
+
filter(lambda matcher: matcher.request is request, self._get_matchers())
|
|
134
|
+
)
|
|
135
|
+
if len(corresponding_matchers) != 1:
|
|
136
|
+
raise ValueError(
|
|
137
|
+
f"Was expecting only one matcher to match the request but got `{corresponding_matchers}`"
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
assert corresponding_matchers[0].actual_number_of_matches == number_of_calls
|
|
141
|
+
|
|
142
|
+
# trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"`
|
|
143
|
+
def __call__(self, f): # type: ignore
|
|
144
|
+
@functools.wraps(f)
|
|
145
|
+
def wrapper(*args, **kwargs): # type: ignore # this is a very generic wrapper that does not need to be typed
|
|
146
|
+
with self:
|
|
147
|
+
assertion_error = None
|
|
148
|
+
|
|
149
|
+
kwargs["http_mocker"] = self
|
|
150
|
+
try:
|
|
151
|
+
result = f(*args, **kwargs)
|
|
152
|
+
except requests_mock.NoMockAddress as no_mock_exception:
|
|
153
|
+
matchers_as_string = "\n\t".join(
|
|
154
|
+
map(lambda matcher: str(matcher.request), self._get_matchers())
|
|
155
|
+
)
|
|
156
|
+
raise ValueError(
|
|
157
|
+
f"No matcher matches {no_mock_exception.args[0]} with headers `{no_mock_exception.request.headers}` "
|
|
158
|
+
f"and body `{no_mock_exception.request.body}`. "
|
|
159
|
+
f"Matchers currently configured are:\n\t{matchers_as_string}."
|
|
160
|
+
) from no_mock_exception
|
|
161
|
+
except AssertionError as test_assertion:
|
|
162
|
+
assertion_error = test_assertion
|
|
163
|
+
|
|
164
|
+
# We validate the matchers before raising the assertion error because we want to show the tester if an HTTP request wasn't
|
|
165
|
+
# mocked correctly
|
|
166
|
+
try:
|
|
167
|
+
self._validate_all_matchers_called()
|
|
168
|
+
except ValueError as http_mocker_exception:
|
|
169
|
+
# This seems useless as it catches ValueError and raises ValueError but without this, the prevailing error message in
|
|
170
|
+
# the output is the function call that failed the assertion, whereas raising `ValueError(http_mocker_exception)`
|
|
171
|
+
# like we do here provides additional context for the exception.
|
|
172
|
+
raise ValueError(http_mocker_exception) from None
|
|
173
|
+
if assertion_error:
|
|
174
|
+
raise assertion_error
|
|
175
|
+
return result
|
|
176
|
+
|
|
177
|
+
return wrapper
|
|
178
|
+
|
|
179
|
+
def _get_matchers(self) -> Iterable[HttpRequestMatcher]:
|
|
180
|
+
for matchers in self._matchers.values():
|
|
181
|
+
yield from matchers
|
|
182
|
+
|
|
183
|
+
def clear_all_matchers(self) -> None:
|
|
184
|
+
"""Clears all stored matchers by resetting the _matchers list to an empty state."""
|
|
185
|
+
self._matchers = defaultdict(list)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from typing import Any, List, Mapping, Optional, Union
|
|
5
|
+
from urllib.parse import parse_qs, urlencode, urlparse
|
|
6
|
+
|
|
7
|
+
ANY_QUERY_PARAMS = "any query_parameters"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _is_subdict(small: Mapping[str, str], big: Mapping[str, str]) -> bool:
|
|
11
|
+
return dict(big, **small) == big
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class HttpRequest:
|
|
15
|
+
def __init__(
|
|
16
|
+
self,
|
|
17
|
+
url: str,
|
|
18
|
+
query_params: Optional[Union[str, Mapping[str, Union[str, List[str]]]]] = None,
|
|
19
|
+
headers: Optional[Mapping[str, str]] = None,
|
|
20
|
+
body: Optional[Union[str, bytes, Mapping[str, Any]]] = None,
|
|
21
|
+
) -> None:
|
|
22
|
+
self._parsed_url = urlparse(url)
|
|
23
|
+
self._query_params = query_params
|
|
24
|
+
if not self._parsed_url.query and query_params:
|
|
25
|
+
self._parsed_url = urlparse(f"{url}?{self._encode_qs(query_params)}")
|
|
26
|
+
elif self._parsed_url.query and query_params:
|
|
27
|
+
raise ValueError(
|
|
28
|
+
"If query params are provided as part of the url, `query_params` should be empty"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
self._headers = headers or {}
|
|
32
|
+
self._body = body
|
|
33
|
+
|
|
34
|
+
@staticmethod
|
|
35
|
+
def _encode_qs(query_params: Union[str, Mapping[str, Union[str, List[str]]]]) -> str:
|
|
36
|
+
if isinstance(query_params, str):
|
|
37
|
+
return query_params
|
|
38
|
+
return urlencode(query_params, doseq=True)
|
|
39
|
+
|
|
40
|
+
def matches(self, other: Any) -> bool:
|
|
41
|
+
"""
|
|
42
|
+
If the body of any request is a Mapping, we compare as Mappings which means that the order is not important.
|
|
43
|
+
If the body is a string, encoding ISO-8859-1 will be assumed
|
|
44
|
+
Headers only need to be a subset of `other` in order to match
|
|
45
|
+
"""
|
|
46
|
+
if isinstance(other, HttpRequest):
|
|
47
|
+
# if `other` is a mapping, we match as an object and formatting is not considers
|
|
48
|
+
if isinstance(self._body, Mapping) or isinstance(other._body, Mapping):
|
|
49
|
+
body_match = self._to_mapping(self._body) == self._to_mapping(other._body)
|
|
50
|
+
else:
|
|
51
|
+
body_match = self._to_bytes(self._body) == self._to_bytes(other._body)
|
|
52
|
+
|
|
53
|
+
return (
|
|
54
|
+
self._parsed_url.scheme == other._parsed_url.scheme
|
|
55
|
+
and self._parsed_url.hostname == other._parsed_url.hostname
|
|
56
|
+
and self._parsed_url.path == other._parsed_url.path
|
|
57
|
+
and (
|
|
58
|
+
ANY_QUERY_PARAMS in (self._query_params, other._query_params)
|
|
59
|
+
or parse_qs(self._parsed_url.query) == parse_qs(other._parsed_url.query)
|
|
60
|
+
)
|
|
61
|
+
and _is_subdict(other._headers, self._headers)
|
|
62
|
+
and body_match
|
|
63
|
+
)
|
|
64
|
+
return False
|
|
65
|
+
|
|
66
|
+
@staticmethod
|
|
67
|
+
def _to_mapping(
|
|
68
|
+
body: Optional[Union[str, bytes, Mapping[str, Any]]],
|
|
69
|
+
) -> Optional[Mapping[str, Any]]:
|
|
70
|
+
if isinstance(body, Mapping):
|
|
71
|
+
return body
|
|
72
|
+
elif isinstance(body, bytes):
|
|
73
|
+
return json.loads(body.decode()) # type: ignore # assumes return type of Mapping[str, Any]
|
|
74
|
+
elif isinstance(body, str):
|
|
75
|
+
return json.loads(body) # type: ignore # assumes return type of Mapping[str, Any]
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
@staticmethod
|
|
79
|
+
def _to_bytes(body: Optional[Union[str, bytes]]) -> bytes:
|
|
80
|
+
if isinstance(body, bytes):
|
|
81
|
+
return body
|
|
82
|
+
elif isinstance(body, str):
|
|
83
|
+
# `ISO-8859-1` is the default encoding used by requests
|
|
84
|
+
return body.encode("ISO-8859-1")
|
|
85
|
+
return b""
|
|
86
|
+
|
|
87
|
+
def __str__(self) -> str:
|
|
88
|
+
return f"{self._parsed_url} with headers {self._headers} and body {self._body!r})"
|
|
89
|
+
|
|
90
|
+
def __repr__(self) -> str:
|
|
91
|
+
return (
|
|
92
|
+
f"HttpRequest(request={self._parsed_url}, headers={self._headers}, body={self._body!r})"
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
def __eq__(self, other: Any) -> bool:
|
|
96
|
+
if isinstance(other, HttpRequest):
|
|
97
|
+
return (
|
|
98
|
+
self._parsed_url == other._parsed_url
|
|
99
|
+
and self._query_params == other._query_params
|
|
100
|
+
and self._headers == other._headers
|
|
101
|
+
and self._body == other._body
|
|
102
|
+
)
|
|
103
|
+
return False
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
2
|
+
|
|
3
|
+
from types import MappingProxyType
|
|
4
|
+
from typing import Mapping, Union
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class HttpResponse:
|
|
8
|
+
def __init__(
|
|
9
|
+
self,
|
|
10
|
+
body: Union[str, bytes],
|
|
11
|
+
status_code: int = 200,
|
|
12
|
+
headers: Mapping[str, str] = MappingProxyType({}),
|
|
13
|
+
):
|
|
14
|
+
self._body = body
|
|
15
|
+
self._status_code = status_code
|
|
16
|
+
self._headers = headers
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
def body(self) -> Union[str, bytes]:
|
|
20
|
+
return self._body
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def status_code(self) -> int:
|
|
24
|
+
return self._status_code
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def headers(self) -> Mapping[str, str]:
|
|
28
|
+
return self._headers
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
2
|
+
|
|
3
|
+
import functools
|
|
4
|
+
import json
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from pathlib import Path as FilePath
|
|
7
|
+
from typing import Any, Dict, List, Optional, Union
|
|
8
|
+
|
|
9
|
+
from airbyte_cdk.test.mock_http.response import HttpResponse
|
|
10
|
+
from airbyte_cdk.test.utils.data import get_unit_test_folder
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _extract(path: List[str], response_template: Dict[str, Any]) -> Any:
|
|
14
|
+
return functools.reduce(lambda a, b: a[b], path, response_template)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _replace_value(dictionary: Dict[str, Any], path: List[str], value: Any) -> None:
|
|
18
|
+
current = dictionary
|
|
19
|
+
for key in path[:-1]:
|
|
20
|
+
current = current[key]
|
|
21
|
+
current[path[-1]] = value
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _write(dictionary: Dict[str, Any], path: List[str], value: Any) -> None:
|
|
25
|
+
current = dictionary
|
|
26
|
+
for key in path[:-1]:
|
|
27
|
+
current = current.setdefault(key, {})
|
|
28
|
+
current[path[-1]] = value
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Path(ABC):
|
|
32
|
+
@abstractmethod
|
|
33
|
+
def write(self, template: Dict[str, Any], value: Any) -> None:
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
@abstractmethod
|
|
37
|
+
def update(self, template: Dict[str, Any], value: Any) -> None:
|
|
38
|
+
pass
|
|
39
|
+
|
|
40
|
+
def extract(self, template: Dict[str, Any]) -> Any:
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class FieldPath(Path):
|
|
45
|
+
def __init__(self, field: str):
|
|
46
|
+
self._path = [field]
|
|
47
|
+
|
|
48
|
+
def write(self, template: Dict[str, Any], value: Any) -> None:
|
|
49
|
+
_write(template, self._path, value)
|
|
50
|
+
|
|
51
|
+
def update(self, template: Dict[str, Any], value: Any) -> None:
|
|
52
|
+
_replace_value(template, self._path, value)
|
|
53
|
+
|
|
54
|
+
def extract(self, template: Dict[str, Any]) -> Any:
|
|
55
|
+
return _extract(self._path, template)
|
|
56
|
+
|
|
57
|
+
def __str__(self) -> str:
|
|
58
|
+
return f"FieldPath(field={self._path[0]})"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class NestedPath(Path):
|
|
62
|
+
def __init__(self, path: List[str]):
|
|
63
|
+
self._path = path
|
|
64
|
+
|
|
65
|
+
def write(self, template: Dict[str, Any], value: Any) -> None:
|
|
66
|
+
_write(template, self._path, value)
|
|
67
|
+
|
|
68
|
+
def update(self, template: Dict[str, Any], value: Any) -> None:
|
|
69
|
+
_replace_value(template, self._path, value)
|
|
70
|
+
|
|
71
|
+
def extract(self, template: Dict[str, Any]) -> Any:
|
|
72
|
+
return _extract(self._path, template)
|
|
73
|
+
|
|
74
|
+
def __str__(self) -> str:
|
|
75
|
+
return f"NestedPath(path={self._path})"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class PaginationStrategy(ABC):
|
|
79
|
+
@abstractmethod
|
|
80
|
+
def update(self, response: Dict[str, Any]) -> None:
|
|
81
|
+
pass
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class FieldUpdatePaginationStrategy(PaginationStrategy):
|
|
85
|
+
def __init__(self, path: Path, value: Any):
|
|
86
|
+
self._path = path
|
|
87
|
+
self._value = value
|
|
88
|
+
|
|
89
|
+
def update(self, response: Dict[str, Any]) -> None:
|
|
90
|
+
self._path.update(response, self._value)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class RecordBuilder:
|
|
94
|
+
def __init__(
|
|
95
|
+
self,
|
|
96
|
+
template: Dict[str, Any],
|
|
97
|
+
id_path: Optional[Path],
|
|
98
|
+
cursor_path: Optional[Union[FieldPath, NestedPath]],
|
|
99
|
+
):
|
|
100
|
+
self._record = template
|
|
101
|
+
self._id_path = id_path
|
|
102
|
+
self._cursor_path = cursor_path
|
|
103
|
+
|
|
104
|
+
self._validate_template()
|
|
105
|
+
|
|
106
|
+
def _validate_template(self) -> None:
|
|
107
|
+
paths_to_validate = [
|
|
108
|
+
("_id_path", self._id_path),
|
|
109
|
+
("_cursor_path", self._cursor_path),
|
|
110
|
+
]
|
|
111
|
+
for field_name, field_path in paths_to_validate:
|
|
112
|
+
self._validate_field(field_name, field_path)
|
|
113
|
+
|
|
114
|
+
def _validate_field(self, field_name: str, path: Optional[Path]) -> None:
|
|
115
|
+
try:
|
|
116
|
+
if path and not path.extract(self._record):
|
|
117
|
+
raise ValueError(
|
|
118
|
+
f"{field_name} `{path}` was provided but it is not part of the template `{self._record}`"
|
|
119
|
+
)
|
|
120
|
+
except (IndexError, KeyError) as exception:
|
|
121
|
+
raise ValueError(
|
|
122
|
+
f"{field_name} `{path}` was provided but it is not part of the template `{self._record}`"
|
|
123
|
+
) from exception
|
|
124
|
+
|
|
125
|
+
def with_id(self, identifier: Any) -> "RecordBuilder":
|
|
126
|
+
self._set_field("id", self._id_path, identifier)
|
|
127
|
+
return self
|
|
128
|
+
|
|
129
|
+
def with_cursor(self, cursor_value: Any) -> "RecordBuilder":
|
|
130
|
+
self._set_field("cursor", self._cursor_path, cursor_value)
|
|
131
|
+
return self
|
|
132
|
+
|
|
133
|
+
def with_field(self, path: Path, value: Any) -> "RecordBuilder":
|
|
134
|
+
path.write(self._record, value)
|
|
135
|
+
return self
|
|
136
|
+
|
|
137
|
+
def _set_field(self, field_name: str, path: Optional[Path], value: Any) -> None:
|
|
138
|
+
if not path:
|
|
139
|
+
raise ValueError(
|
|
140
|
+
f"{field_name}_path was not provided and hence, the record {field_name} can't be modified. Please provide `id_field` while "
|
|
141
|
+
f"instantiating RecordBuilder to leverage this capability"
|
|
142
|
+
)
|
|
143
|
+
path.update(self._record, value)
|
|
144
|
+
|
|
145
|
+
def build(self) -> Dict[str, Any]:
|
|
146
|
+
return self._record
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class HttpResponseBuilder:
|
|
150
|
+
def __init__(
|
|
151
|
+
self,
|
|
152
|
+
template: Dict[str, Any],
|
|
153
|
+
records_path: Union[FieldPath, NestedPath],
|
|
154
|
+
pagination_strategy: Optional[PaginationStrategy],
|
|
155
|
+
):
|
|
156
|
+
self._response = template
|
|
157
|
+
self._records: List[RecordBuilder] = []
|
|
158
|
+
self._records_path = records_path
|
|
159
|
+
self._pagination_strategy = pagination_strategy
|
|
160
|
+
self._status_code = 200
|
|
161
|
+
|
|
162
|
+
def with_record(self, record: RecordBuilder) -> "HttpResponseBuilder":
|
|
163
|
+
self._records.append(record)
|
|
164
|
+
return self
|
|
165
|
+
|
|
166
|
+
def with_pagination(self) -> "HttpResponseBuilder":
|
|
167
|
+
if not self._pagination_strategy:
|
|
168
|
+
raise ValueError(
|
|
169
|
+
"`pagination_strategy` was not provided and hence, fields related to the pagination can't be modified. Please provide "
|
|
170
|
+
"`pagination_strategy` while instantiating ResponseBuilder to leverage this capability"
|
|
171
|
+
)
|
|
172
|
+
self._pagination_strategy.update(self._response)
|
|
173
|
+
return self
|
|
174
|
+
|
|
175
|
+
def with_status_code(self, status_code: int) -> "HttpResponseBuilder":
|
|
176
|
+
self._status_code = status_code
|
|
177
|
+
return self
|
|
178
|
+
|
|
179
|
+
def build(self) -> HttpResponse:
|
|
180
|
+
self._records_path.update(self._response, [record.build() for record in self._records])
|
|
181
|
+
return HttpResponse(json.dumps(self._response), self._status_code)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _get_unit_test_folder(execution_folder: str) -> FilePath:
|
|
185
|
+
# FIXME: This function should be removed after the next CDK release to avoid breaking amazon-seller-partner test code.
|
|
186
|
+
return get_unit_test_folder(execution_folder)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def find_template(resource: str, execution_folder: str) -> Dict[str, Any]:
|
|
190
|
+
response_template_filepath = str(
|
|
191
|
+
get_unit_test_folder(execution_folder)
|
|
192
|
+
/ "resource"
|
|
193
|
+
/ "http"
|
|
194
|
+
/ "response"
|
|
195
|
+
/ f"{resource}.json"
|
|
196
|
+
)
|
|
197
|
+
with open(response_template_filepath, "r") as template_file:
|
|
198
|
+
return json.load(template_file) # type: ignore # we assume the dev correctly set up the resource file
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def find_binary_response(resource: str, execution_folder: str) -> bytes:
|
|
202
|
+
response_filepath = str(
|
|
203
|
+
get_unit_test_folder(execution_folder) / "resource" / "http" / "response" / f"{resource}"
|
|
204
|
+
)
|
|
205
|
+
with open(response_filepath, "rb") as response_file:
|
|
206
|
+
return response_file.read() # type: ignore # we assume the dev correctly set up the resource file
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def create_record_builder(
|
|
210
|
+
response_template: Dict[str, Any],
|
|
211
|
+
records_path: Union[FieldPath, NestedPath],
|
|
212
|
+
record_id_path: Optional[Path] = None,
|
|
213
|
+
record_cursor_path: Optional[Union[FieldPath, NestedPath]] = None,
|
|
214
|
+
) -> RecordBuilder:
|
|
215
|
+
"""
|
|
216
|
+
This will use the first record define at `records_path` as a template for the records. If more records are defined, they will be ignored
|
|
217
|
+
"""
|
|
218
|
+
try:
|
|
219
|
+
record_template = records_path.extract(response_template)[0]
|
|
220
|
+
if not record_template:
|
|
221
|
+
raise ValueError(
|
|
222
|
+
f"Could not extract any record from template at path `{records_path}`. "
|
|
223
|
+
f"Please fix the template to provide a record sample or fix `records_path`."
|
|
224
|
+
)
|
|
225
|
+
return RecordBuilder(record_template, record_id_path, record_cursor_path)
|
|
226
|
+
except (IndexError, KeyError):
|
|
227
|
+
raise ValueError(
|
|
228
|
+
f"Error while extracting records at path `{records_path}` from response template `{response_template}`"
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def create_response_builder(
|
|
233
|
+
response_template: Dict[str, Any],
|
|
234
|
+
records_path: Union[FieldPath, NestedPath],
|
|
235
|
+
pagination_strategy: Optional[PaginationStrategy] = None,
|
|
236
|
+
) -> HttpResponseBuilder:
|
|
237
|
+
return HttpResponseBuilder(response_template, records_path, pagination_strategy)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
2
|
+
|
|
3
|
+
from typing import Any, List
|
|
4
|
+
|
|
5
|
+
from airbyte_cdk.models import (
|
|
6
|
+
AirbyteStateBlob,
|
|
7
|
+
AirbyteStateMessage,
|
|
8
|
+
AirbyteStateType,
|
|
9
|
+
AirbyteStreamState,
|
|
10
|
+
StreamDescriptor,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class StateBuilder:
|
|
15
|
+
def __init__(self) -> None:
|
|
16
|
+
self._state: List[AirbyteStateMessage] = []
|
|
17
|
+
|
|
18
|
+
def with_stream_state(self, stream_name: str, state: Any) -> "StateBuilder":
|
|
19
|
+
self._state.append(
|
|
20
|
+
AirbyteStateMessage(
|
|
21
|
+
type=AirbyteStateType.STREAM,
|
|
22
|
+
stream=AirbyteStreamState(
|
|
23
|
+
stream_state=state
|
|
24
|
+
if isinstance(state, AirbyteStateBlob)
|
|
25
|
+
else AirbyteStateBlob(state),
|
|
26
|
+
stream_descriptor=StreamDescriptor(**{"name": stream_name}),
|
|
27
|
+
),
|
|
28
|
+
)
|
|
29
|
+
)
|
|
30
|
+
return self
|
|
31
|
+
|
|
32
|
+
def build(self) -> List[AirbyteStateMessage]:
|
|
33
|
+
return self._state
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
|
2
|
+
|
|
3
|
+
from pydantic import FilePath
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_unit_test_folder(execution_folder: str) -> FilePath:
|
|
7
|
+
path = FilePath(execution_folder)
|
|
8
|
+
while path.name != "unit_tests":
|
|
9
|
+
if path.name == path.root or path.name == path.drive:
|
|
10
|
+
raise ValueError(
|
|
11
|
+
f"Could not find `unit_tests` folder as a parent of {execution_folder}"
|
|
12
|
+
)
|
|
13
|
+
path = path.parent
|
|
14
|
+
return path
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def read_resource_file_contents(resource: str, test_location: str) -> str:
|
|
18
|
+
"""Read the contents of a test data file from the test resource folder."""
|
|
19
|
+
file_path = str(
|
|
20
|
+
get_unit_test_folder(test_location) / "resource" / "http" / "response" / f"{resource}"
|
|
21
|
+
)
|
|
22
|
+
with open(file_path) as f:
|
|
23
|
+
response = f.read()
|
|
24
|
+
return response
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from typing import Any, Mapping
|
|
5
|
+
|
|
6
|
+
from requests_mock import Mocker
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def register_mock_responses(
|
|
10
|
+
mocker: Mocker, http_calls: list[Mapping[str, Mapping[str, Any]]]
|
|
11
|
+
) -> None:
|
|
12
|
+
"""Register a list of HTTP request-response pairs."""
|
|
13
|
+
for call in http_calls:
|
|
14
|
+
request, response = call["request"], call["response"]
|
|
15
|
+
matcher = re.compile(request["url"]) if request["is_regex"] else request["url"]
|
|
16
|
+
mocker.register_uri(request["method"], matcher, **response)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
import importlib.util
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from types import ModuleType
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
# The following fixtures are used to load a manifest-only connector's components module and manifest file.
|
|
11
|
+
# They can be accessed from any test file in the connector's unit_tests directory by importing them as follows:
|
|
12
|
+
|
|
13
|
+
# from airbyte_cdk.test.utils.manifest_only_fixtures import components_module, connector_dir, manifest_path
|
|
14
|
+
|
|
15
|
+
# individual components can then be referenced as: components_module.<CustomComponentClass>
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@pytest.fixture(scope="session")
|
|
19
|
+
def connector_dir(request: pytest.FixtureRequest) -> Path:
|
|
20
|
+
"""Return the connector's root directory."""
|
|
21
|
+
|
|
22
|
+
current_dir = Path(request.config.invocation_params.dir)
|
|
23
|
+
|
|
24
|
+
# If the tests are run locally from the connector's unit_tests directory, return the parent (connector) directory
|
|
25
|
+
if current_dir.name == "unit_tests":
|
|
26
|
+
return current_dir.parent
|
|
27
|
+
# In CI, the tests are run from the connector directory itself
|
|
28
|
+
return current_dir
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.fixture(scope="session")
|
|
32
|
+
def components_module(connector_dir: Path) -> ModuleType | None:
|
|
33
|
+
"""Load and return the components module from the connector directory.
|
|
34
|
+
|
|
35
|
+
This assumes the components module is located at <connector_dir>/components.py.
|
|
36
|
+
"""
|
|
37
|
+
components_path = connector_dir / "components.py"
|
|
38
|
+
if not components_path.exists():
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
components_spec = importlib.util.spec_from_file_location("components", components_path)
|
|
42
|
+
if components_spec is None:
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
components_module = importlib.util.module_from_spec(components_spec)
|
|
46
|
+
if components_spec.loader is None:
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
components_spec.loader.exec_module(components_module)
|
|
50
|
+
return components_module
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@pytest.fixture(scope="session")
|
|
54
|
+
def manifest_path(connector_dir: Path) -> Path:
|
|
55
|
+
"""Return the path to the connector's manifest file."""
|
|
56
|
+
path = connector_dir / "manifest.yaml"
|
|
57
|
+
if not path.exists():
|
|
58
|
+
raise FileNotFoundError(f"Manifest file not found at {path}")
|
|
59
|
+
return path
|