airbyte-cdk 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/__init__.py +358 -0
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +236 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +104 -0
- airbyte_cdk/connector.py +123 -0
- airbyte_cdk/connector_builder/README.md +53 -0
- airbyte_cdk/connector_builder/__init__.py +3 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +121 -0
- airbyte_cdk/connector_builder/main.py +107 -0
- airbyte_cdk/connector_builder/models.py +73 -0
- airbyte_cdk/connector_builder/test_reader/__init__.py +7 -0
- airbyte_cdk/connector_builder/test_reader/helpers.py +689 -0
- airbyte_cdk/connector_builder/test_reader/message_grouper.py +173 -0
- airbyte_cdk/connector_builder/test_reader/reader.py +441 -0
- airbyte_cdk/connector_builder/test_reader/types.py +83 -0
- airbyte_cdk/destinations/__init__.py +8 -0
- airbyte_cdk/destinations/destination.py +154 -0
- airbyte_cdk/destinations/vector_db_based/README.md +37 -0
- airbyte_cdk/destinations/vector_db_based/__init__.py +38 -0
- airbyte_cdk/destinations/vector_db_based/config.py +298 -0
- airbyte_cdk/destinations/vector_db_based/document_processor.py +223 -0
- airbyte_cdk/destinations/vector_db_based/embedder.py +303 -0
- airbyte_cdk/destinations/vector_db_based/indexer.py +78 -0
- airbyte_cdk/destinations/vector_db_based/test_utils.py +63 -0
- airbyte_cdk/destinations/vector_db_based/utils.py +35 -0
- airbyte_cdk/destinations/vector_db_based/writer.py +104 -0
- airbyte_cdk/entrypoint.py +414 -0
- airbyte_cdk/exception_handler.py +56 -0
- airbyte_cdk/logger.py +109 -0
- airbyte_cdk/models/__init__.py +72 -0
- airbyte_cdk/models/airbyte_protocol.py +88 -0
- airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
- airbyte_cdk/models/well_known_types.py +5 -0
- airbyte_cdk/py.typed +0 -0
- airbyte_cdk/sources/__init__.py +26 -0
- airbyte_cdk/sources/abstract_source.py +326 -0
- airbyte_cdk/sources/concurrent_source/__init__.py +8 -0
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +255 -0
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +165 -0
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +147 -0
- airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py +24 -0
- airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +115 -0
- airbyte_cdk/sources/config.py +27 -0
- airbyte_cdk/sources/connector_state_manager.py +161 -0
- airbyte_cdk/sources/declarative/__init__.py +3 -0
- airbyte_cdk/sources/declarative/async_job/__init__.py +0 -0
- airbyte_cdk/sources/declarative/async_job/job.py +52 -0
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +525 -0
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +79 -0
- airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
- airbyte_cdk/sources/declarative/async_job/status.py +24 -0
- airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
- airbyte_cdk/sources/declarative/auth/__init__.py +8 -0
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +42 -0
- airbyte_cdk/sources/declarative/auth/jwt.py +197 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +293 -0
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +45 -0
- airbyte_cdk/sources/declarative/auth/token.py +267 -0
- airbyte_cdk/sources/declarative/auth/token_provider.py +82 -0
- airbyte_cdk/sources/declarative/checks/__init__.py +24 -0
- airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +61 -0
- airbyte_cdk/sources/declarative/checks/check_stream.py +56 -0
- airbyte_cdk/sources/declarative/checks/connection_checker.py +35 -0
- airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +526 -0
- airbyte_cdk/sources/declarative/datetime/__init__.py +3 -0
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +65 -0
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +118 -0
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +3975 -0
- airbyte_cdk/sources/declarative/declarative_source.py +36 -0
- airbyte_cdk/sources/declarative/declarative_stream.py +241 -0
- airbyte_cdk/sources/declarative/decoders/__init__.py +33 -0
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +218 -0
- airbyte_cdk/sources/declarative/decoders/decoder.py +32 -0
- airbyte_cdk/sources/declarative/decoders/decoder_parser.py +30 -0
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +65 -0
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
- airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py +56 -0
- airbyte_cdk/sources/declarative/exceptions.py +9 -0
- airbyte_cdk/sources/declarative/extractors/__init__.py +21 -0
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +86 -0
- airbyte_cdk/sources/declarative/extractors/http_selector.py +37 -0
- airbyte_cdk/sources/declarative/extractors/record_extractor.py +27 -0
- airbyte_cdk/sources/declarative/extractors/record_filter.py +91 -0
- airbyte_cdk/sources/declarative/extractors/record_selector.py +170 -0
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +176 -0
- airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
- airbyte_cdk/sources/declarative/incremental/__init__.py +37 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +497 -0
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +459 -0
- airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +357 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +380 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
- airbyte_cdk/sources/declarative/interpolation/__init__.py +9 -0
- airbyte_cdk/sources/declarative/interpolation/filters.py +139 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +66 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +56 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +52 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +79 -0
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +34 -0
- airbyte_cdk/sources/declarative/interpolation/jinja.py +161 -0
- airbyte_cdk/sources/declarative/interpolation/macros.py +191 -0
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +421 -0
- airbyte_cdk/sources/declarative/migrations/__init__.py +0 -0
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
- airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
- airbyte_cdk/sources/declarative/models/__init__.py +2 -0
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2503 -0
- airbyte_cdk/sources/declarative/parsers/__init__.py +3 -0
- airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +157 -0
- airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +21 -0
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +172 -0
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +213 -0
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +3407 -0
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +29 -0
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +121 -0
- airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
- airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +63 -0
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +437 -0
- airbyte_cdk/sources/declarative/requesters/README.md +56 -0
- airbyte_cdk/sources/declarative/requesters/__init__.py +9 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +25 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +23 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +45 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +45 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +41 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +70 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +77 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +17 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +101 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +147 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +17 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +179 -0
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +350 -0
- airbyte_cdk/sources/declarative/requesters/http_requester.py +433 -0
- airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +21 -0
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +327 -0
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +76 -0
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +65 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +25 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +98 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +102 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +71 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +48 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +66 -0
- airbyte_cdk/sources/declarative/requesters/request_option.py +117 -0
- airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +23 -0
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +92 -0
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +59 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +68 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +119 -0
- airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +79 -0
- airbyte_cdk/sources/declarative/requesters/request_path.py +15 -0
- airbyte_cdk/sources/declarative/requesters/requester.py +144 -0
- airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
- airbyte_cdk/sources/declarative/retrievers/__init__.py +19 -0
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +124 -0
- airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
- airbyte_cdk/sources/declarative/retrievers/retriever.py +54 -0
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +702 -0
- airbyte_cdk/sources/declarative/schema/__init__.py +25 -0
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +47 -0
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +285 -0
- airbyte_cdk/sources/declarative/schema/inline_schema_loader.py +19 -0
- airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +92 -0
- airbyte_cdk/sources/declarative/schema/schema_loader.py +17 -0
- airbyte_cdk/sources/declarative/spec/__init__.py +7 -0
- airbyte_cdk/sources/declarative/spec/spec.py +48 -0
- airbyte_cdk/sources/declarative/stream_slicers/__init__.py +7 -0
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +93 -0
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +25 -0
- airbyte_cdk/sources/declarative/transformations/__init__.py +17 -0
- airbyte_cdk/sources/declarative/transformations/add_fields.py +146 -0
- airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py +61 -0
- airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
- airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +75 -0
- airbyte_cdk/sources/declarative/transformations/transformation.py +37 -0
- airbyte_cdk/sources/declarative/types.py +25 -0
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +67 -0
- airbyte_cdk/sources/file_based/README.md +152 -0
- airbyte_cdk/sources/file_based/__init__.py +24 -0
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +11 -0
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +73 -0
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +149 -0
- airbyte_cdk/sources/file_based/config/__init__.py +0 -0
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +153 -0
- airbyte_cdk/sources/file_based/config/avro_format.py +25 -0
- airbyte_cdk/sources/file_based/config/csv_format.py +210 -0
- airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +99 -0
- airbyte_cdk/sources/file_based/config/jsonl_format.py +18 -0
- airbyte_cdk/sources/file_based/config/parquet_format.py +25 -0
- airbyte_cdk/sources/file_based/config/unstructured_format.py +102 -0
- airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py +81 -0
- airbyte_cdk/sources/file_based/discovery_policy/__init__.py +8 -0
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +21 -0
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +33 -0
- airbyte_cdk/sources/file_based/exceptions.py +159 -0
- airbyte_cdk/sources/file_based/file_based_source.py +466 -0
- airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py +123 -0
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +209 -0
- airbyte_cdk/sources/file_based/file_record_data.py +22 -0
- airbyte_cdk/sources/file_based/file_types/__init__.py +37 -0
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +233 -0
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +527 -0
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +30 -0
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +86 -0
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +145 -0
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +275 -0
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +480 -0
- airbyte_cdk/sources/file_based/remote_file.py +18 -0
- airbyte_cdk/sources/file_based/schema_helpers.py +281 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +17 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +20 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +52 -0
- airbyte_cdk/sources/file_based/stream/__init__.py +13 -0
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +197 -0
- airbyte_cdk/sources/file_based/stream/concurrent/__init__.py +0 -0
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +343 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +9 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +59 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +313 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +83 -0
- airbyte_cdk/sources/file_based/stream/cursor/__init__.py +4 -0
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +66 -0
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +149 -0
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +396 -0
- airbyte_cdk/sources/file_based/stream/identities_stream.py +49 -0
- airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +92 -0
- airbyte_cdk/sources/file_based/types.py +10 -0
- airbyte_cdk/sources/http_config.py +10 -0
- airbyte_cdk/sources/http_logger.py +55 -0
- airbyte_cdk/sources/message/__init__.py +19 -0
- airbyte_cdk/sources/message/repository.py +137 -0
- airbyte_cdk/sources/source.py +95 -0
- airbyte_cdk/sources/specs/transfer_modes.py +26 -0
- airbyte_cdk/sources/streams/__init__.py +8 -0
- airbyte_cdk/sources/streams/availability_strategy.py +84 -0
- airbyte_cdk/sources/streams/call_rate.py +704 -0
- airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
- airbyte_cdk/sources/streams/checkpoint/cursor.py +77 -0
- airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
- airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
- airbyte_cdk/sources/streams/concurrent/README.md +7 -0
- airbyte_cdk/sources/streams/concurrent/__init__.py +3 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +96 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py +37 -0
- airbyte_cdk/sources/streams/concurrent/adapters.py +397 -0
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +94 -0
- airbyte_cdk/sources/streams/concurrent/clamping.py +99 -0
- airbyte_cdk/sources/streams/concurrent/cursor.py +481 -0
- airbyte_cdk/sources/streams/concurrent/cursor_types.py +32 -0
- airbyte_cdk/sources/streams/concurrent/default_stream.py +102 -0
- airbyte_cdk/sources/streams/concurrent/exceptions.py +18 -0
- airbyte_cdk/sources/streams/concurrent/helpers.py +42 -0
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +64 -0
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +45 -0
- airbyte_cdk/sources/streams/concurrent/partitions/__init__.py +3 -0
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +48 -0
- airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py +18 -0
- airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +38 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py +0 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +182 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +223 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
- airbyte_cdk/sources/streams/core.py +703 -0
- airbyte_cdk/sources/streams/http/__init__.py +10 -0
- airbyte_cdk/sources/streams/http/availability_strategy.py +54 -0
- airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
- airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
- airbyte_cdk/sources/streams/http/exceptions.py +61 -0
- airbyte_cdk/sources/streams/http/http.py +673 -0
- airbyte_cdk/sources/streams/http/http_client.py +531 -0
- airbyte_cdk/sources/streams/http/rate_limiting.py +158 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py +14 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +479 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +34 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +436 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +83 -0
- airbyte_cdk/sources/streams/permissions/identities_stream.py +75 -0
- airbyte_cdk/sources/streams/utils/__init__.py +3 -0
- airbyte_cdk/sources/types.py +169 -0
- airbyte_cdk/sources/utils/__init__.py +7 -0
- airbyte_cdk/sources/utils/casing.py +12 -0
- airbyte_cdk/sources/utils/files_directory.py +15 -0
- airbyte_cdk/sources/utils/record_helper.py +53 -0
- airbyte_cdk/sources/utils/schema_helpers.py +230 -0
- airbyte_cdk/sources/utils/slice_logger.py +57 -0
- airbyte_cdk/sources/utils/transform.py +277 -0
- airbyte_cdk/sources/utils/types.py +7 -0
- airbyte_cdk/sql/__init__.py +0 -0
- airbyte_cdk/sql/_util/__init__.py +0 -0
- airbyte_cdk/sql/_util/hashing.py +34 -0
- airbyte_cdk/sql/_util/name_normalizers.py +92 -0
- airbyte_cdk/sql/constants.py +32 -0
- airbyte_cdk/sql/exceptions.py +235 -0
- airbyte_cdk/sql/secrets.py +123 -0
- airbyte_cdk/sql/shared/__init__.py +15 -0
- airbyte_cdk/sql/shared/catalog_providers.py +145 -0
- airbyte_cdk/sql/shared/sql_processor.py +786 -0
- airbyte_cdk/sql/types.py +160 -0
- airbyte_cdk/test/__init__.py +7 -0
- airbyte_cdk/test/catalog_builder.py +81 -0
- airbyte_cdk/test/entrypoint_wrapper.py +250 -0
- airbyte_cdk/test/mock_http/__init__.py +6 -0
- airbyte_cdk/test/mock_http/matcher.py +41 -0
- airbyte_cdk/test/mock_http/mocker.py +185 -0
- airbyte_cdk/test/mock_http/request.py +103 -0
- airbyte_cdk/test/mock_http/response.py +28 -0
- airbyte_cdk/test/mock_http/response_builder.py +237 -0
- airbyte_cdk/test/state_builder.py +33 -0
- airbyte_cdk/test/utils/__init__.py +1 -0
- airbyte_cdk/test/utils/data.py +24 -0
- airbyte_cdk/test/utils/http_mocking.py +16 -0
- airbyte_cdk/test/utils/manifest_only_fixtures.py +59 -0
- airbyte_cdk/test/utils/reading.py +26 -0
- airbyte_cdk/utils/__init__.py +10 -0
- airbyte_cdk/utils/airbyte_secrets_utils.py +80 -0
- airbyte_cdk/utils/analytics_message.py +25 -0
- airbyte_cdk/utils/constants.py +5 -0
- airbyte_cdk/utils/datetime_format_inferrer.py +94 -0
- airbyte_cdk/utils/datetime_helpers.py +499 -0
- airbyte_cdk/utils/event_timing.py +85 -0
- airbyte_cdk/utils/is_cloud_environment.py +18 -0
- airbyte_cdk/utils/mapping_helpers.py +162 -0
- airbyte_cdk/utils/message_utils.py +26 -0
- airbyte_cdk/utils/oneof_option_config.py +33 -0
- airbyte_cdk/utils/print_buffer.py +75 -0
- airbyte_cdk/utils/schema_inferrer.py +270 -0
- airbyte_cdk/utils/slice_hasher.py +37 -0
- airbyte_cdk/utils/spec_schema_transformations.py +26 -0
- airbyte_cdk/utils/stream_status_utils.py +43 -0
- airbyte_cdk/utils/traced_exception.py +145 -0
- airbyte_cdk-0.0.0.dev0.dist-info/LICENSE.txt +19 -0
- airbyte_cdk-0.0.0.dev0.dist-info/LICENSE_SHORT +1 -0
- airbyte_cdk-0.0.0.dev0.dist-info/METADATA +111 -0
- airbyte_cdk-0.0.0.dev0.dist-info/RECORD +368 -0
- airbyte_cdk-0.0.0.dev0.dist-info/WHEEL +4 -0
- airbyte_cdk-0.0.0.dev0.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
|
2
|
+
from datetime import datetime, timedelta, timezone
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Timer:
|
|
7
|
+
def __init__(self, timeout: timedelta) -> None:
|
|
8
|
+
self._start_datetime: Optional[datetime] = None
|
|
9
|
+
self._end_datetime: Optional[datetime] = None
|
|
10
|
+
self._timeout = timeout
|
|
11
|
+
|
|
12
|
+
def start(self) -> None:
|
|
13
|
+
self._start_datetime = self._now()
|
|
14
|
+
self._end_datetime = None
|
|
15
|
+
|
|
16
|
+
def stop(self) -> None:
|
|
17
|
+
if self._end_datetime is None:
|
|
18
|
+
self._end_datetime = self._now()
|
|
19
|
+
|
|
20
|
+
def is_started(self) -> bool:
|
|
21
|
+
return self._start_datetime is not None
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def elapsed_time(self) -> Optional[timedelta]:
|
|
25
|
+
if not self._start_datetime:
|
|
26
|
+
return None
|
|
27
|
+
|
|
28
|
+
end_time = self._end_datetime or self._now()
|
|
29
|
+
elapsed_period = end_time - self._start_datetime
|
|
30
|
+
return elapsed_period
|
|
31
|
+
|
|
32
|
+
def has_timed_out(self) -> bool:
|
|
33
|
+
if not self.is_started():
|
|
34
|
+
return False
|
|
35
|
+
return self.elapsed_time > self._timeout # type: ignore # given the job timer is started, we assume there is an elapsed_period
|
|
36
|
+
|
|
37
|
+
@staticmethod
|
|
38
|
+
def _now() -> datetime:
|
|
39
|
+
return datetime.now(tz=timezone.utc)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from airbyte_cdk.sources.declarative.auth.jwt import JwtAuthenticator
|
|
6
|
+
from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeOauth2Authenticator
|
|
7
|
+
|
|
8
|
+
__all__ = ["DeclarativeOauth2Authenticator", "JwtAuthenticator"]
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from dataclasses import InitVar, dataclass
|
|
6
|
+
from typing import Any, Mapping, Union
|
|
7
|
+
|
|
8
|
+
from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import (
|
|
9
|
+
AbstractHeaderAuthenticator,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class DeclarativeAuthenticator(AbstractHeaderAuthenticator):
|
|
15
|
+
"""
|
|
16
|
+
Interface used to associate which authenticators can be used as part of the declarative framework
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def get_request_params(self) -> Mapping[str, Any]:
|
|
20
|
+
"""HTTP request parameter to add to the requests"""
|
|
21
|
+
return {}
|
|
22
|
+
|
|
23
|
+
def get_request_body_data(self) -> Union[Mapping[str, Any], str]:
|
|
24
|
+
"""Form-encoded body data to set on the requests"""
|
|
25
|
+
return {}
|
|
26
|
+
|
|
27
|
+
def get_request_body_json(self) -> Mapping[str, Any]:
|
|
28
|
+
"""JSON-encoded body data to set on the requests"""
|
|
29
|
+
return {}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class NoAuth(DeclarativeAuthenticator):
|
|
34
|
+
parameters: InitVar[Mapping[str, Any]]
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def auth_header(self) -> str:
|
|
38
|
+
return ""
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def token(self) -> str:
|
|
42
|
+
return ""
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import json
|
|
7
|
+
from dataclasses import InitVar, dataclass
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from typing import Any, Mapping, Optional, Union
|
|
10
|
+
|
|
11
|
+
import jwt
|
|
12
|
+
|
|
13
|
+
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
|
|
14
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
|
15
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
|
|
16
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class JwtAlgorithm(str):
|
|
20
|
+
"""
|
|
21
|
+
Enum for supported JWT algorithms
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
HS256 = "HS256"
|
|
25
|
+
HS384 = "HS384"
|
|
26
|
+
HS512 = "HS512"
|
|
27
|
+
ES256 = "ES256"
|
|
28
|
+
ES256K = "ES256K"
|
|
29
|
+
ES384 = "ES384"
|
|
30
|
+
ES512 = "ES512"
|
|
31
|
+
RS256 = "RS256"
|
|
32
|
+
RS384 = "RS384"
|
|
33
|
+
RS512 = "RS512"
|
|
34
|
+
PS256 = "PS256"
|
|
35
|
+
PS384 = "PS384"
|
|
36
|
+
PS512 = "PS512"
|
|
37
|
+
EdDSA = "EdDSA"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class JwtAuthenticator(DeclarativeAuthenticator):
|
|
42
|
+
"""
|
|
43
|
+
Generates a JSON Web Token (JWT) based on a declarative connector configuration file. The generated token is attached to each request via the Authorization header.
|
|
44
|
+
|
|
45
|
+
Attributes:
|
|
46
|
+
config (Mapping[str, Any]): The user-provided configuration as specified by the source's spec
|
|
47
|
+
secret_key (Union[InterpolatedString, str]): The secret key used to sign the JWT
|
|
48
|
+
algorithm (Union[str, JwtAlgorithm]): The algorithm used to sign the JWT
|
|
49
|
+
token_duration (Optional[int]): The duration in seconds for which the token is valid
|
|
50
|
+
base64_encode_secret_key (Optional[Union[InterpolatedBoolean, str, bool]]): Whether to base64 encode the secret key
|
|
51
|
+
header_prefix (Optional[Union[InterpolatedString, str]]): The prefix to add to the Authorization header
|
|
52
|
+
kid (Optional[Union[InterpolatedString, str]]): The key identifier to be included in the JWT header
|
|
53
|
+
typ (Optional[Union[InterpolatedString, str]]): The type of the JWT.
|
|
54
|
+
cty (Optional[Union[InterpolatedString, str]]): The content type of the JWT.
|
|
55
|
+
iss (Optional[Union[InterpolatedString, str]]): The issuer of the JWT.
|
|
56
|
+
sub (Optional[Union[InterpolatedString, str]]): The subject of the JWT.
|
|
57
|
+
aud (Optional[Union[InterpolatedString, str]]): The audience of the JWT.
|
|
58
|
+
additional_jwt_headers (Optional[Mapping[str, Any]]): Additional headers to include in the JWT.
|
|
59
|
+
additional_jwt_payload (Optional[Mapping[str, Any]]): Additional payload to include in the JWT.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
config: Mapping[str, Any]
|
|
63
|
+
parameters: InitVar[Mapping[str, Any]]
|
|
64
|
+
secret_key: Union[InterpolatedString, str]
|
|
65
|
+
algorithm: Union[str, JwtAlgorithm]
|
|
66
|
+
token_duration: Optional[int]
|
|
67
|
+
base64_encode_secret_key: Optional[Union[InterpolatedBoolean, str, bool]] = False
|
|
68
|
+
header_prefix: Optional[Union[InterpolatedString, str]] = None
|
|
69
|
+
kid: Optional[Union[InterpolatedString, str]] = None
|
|
70
|
+
typ: Optional[Union[InterpolatedString, str]] = None
|
|
71
|
+
cty: Optional[Union[InterpolatedString, str]] = None
|
|
72
|
+
iss: Optional[Union[InterpolatedString, str]] = None
|
|
73
|
+
sub: Optional[Union[InterpolatedString, str]] = None
|
|
74
|
+
aud: Optional[Union[InterpolatedString, str]] = None
|
|
75
|
+
additional_jwt_headers: Optional[Mapping[str, Any]] = None
|
|
76
|
+
additional_jwt_payload: Optional[Mapping[str, Any]] = None
|
|
77
|
+
|
|
78
|
+
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
|
79
|
+
self._secret_key = InterpolatedString.create(self.secret_key, parameters=parameters)
|
|
80
|
+
self._algorithm = (
|
|
81
|
+
JwtAlgorithm(self.algorithm) if isinstance(self.algorithm, str) else self.algorithm
|
|
82
|
+
)
|
|
83
|
+
self._base64_encode_secret_key = (
|
|
84
|
+
InterpolatedBoolean(self.base64_encode_secret_key, parameters=parameters)
|
|
85
|
+
if isinstance(self.base64_encode_secret_key, str)
|
|
86
|
+
else self.base64_encode_secret_key
|
|
87
|
+
)
|
|
88
|
+
self._token_duration = self.token_duration
|
|
89
|
+
self._header_prefix = (
|
|
90
|
+
InterpolatedString.create(self.header_prefix, parameters=parameters)
|
|
91
|
+
if self.header_prefix
|
|
92
|
+
else None
|
|
93
|
+
)
|
|
94
|
+
self._kid = InterpolatedString.create(self.kid, parameters=parameters) if self.kid else None
|
|
95
|
+
self._typ = InterpolatedString.create(self.typ, parameters=parameters) if self.typ else None
|
|
96
|
+
self._cty = InterpolatedString.create(self.cty, parameters=parameters) if self.cty else None
|
|
97
|
+
self._iss = InterpolatedString.create(self.iss, parameters=parameters) if self.iss else None
|
|
98
|
+
self._sub = InterpolatedString.create(self.sub, parameters=parameters) if self.sub else None
|
|
99
|
+
self._aud = InterpolatedString.create(self.aud, parameters=parameters) if self.aud else None
|
|
100
|
+
self._additional_jwt_headers = InterpolatedMapping(
|
|
101
|
+
self.additional_jwt_headers or {}, parameters=parameters
|
|
102
|
+
)
|
|
103
|
+
self._additional_jwt_payload = InterpolatedMapping(
|
|
104
|
+
self.additional_jwt_payload or {}, parameters=parameters
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def _get_jwt_headers(self) -> dict[str, Any]:
|
|
108
|
+
"""
|
|
109
|
+
Builds and returns the headers used when signing the JWT.
|
|
110
|
+
"""
|
|
111
|
+
headers = self._additional_jwt_headers.eval(self.config, json_loads=json.loads)
|
|
112
|
+
if any(prop in headers for prop in ["kid", "alg", "typ", "cty"]):
|
|
113
|
+
raise ValueError(
|
|
114
|
+
"'kid', 'alg', 'typ', 'cty' are reserved headers and should not be set as part of 'additional_jwt_headers'"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
if self._kid:
|
|
118
|
+
headers["kid"] = self._kid.eval(self.config, json_loads=json.loads)
|
|
119
|
+
if self._typ:
|
|
120
|
+
headers["typ"] = self._typ.eval(self.config, json_loads=json.loads)
|
|
121
|
+
if self._cty:
|
|
122
|
+
headers["cty"] = self._cty.eval(self.config, json_loads=json.loads)
|
|
123
|
+
headers["alg"] = self._algorithm
|
|
124
|
+
return headers
|
|
125
|
+
|
|
126
|
+
def _get_jwt_payload(self) -> dict[str, Any]:
|
|
127
|
+
"""
|
|
128
|
+
Builds and returns the payload used when signing the JWT.
|
|
129
|
+
"""
|
|
130
|
+
now = int(datetime.now().timestamp())
|
|
131
|
+
exp = now + self._token_duration if isinstance(self._token_duration, int) else now
|
|
132
|
+
nbf = now
|
|
133
|
+
|
|
134
|
+
payload = self._additional_jwt_payload.eval(self.config, json_loads=json.loads)
|
|
135
|
+
if any(prop in payload for prop in ["iss", "sub", "aud", "iat", "exp", "nbf"]):
|
|
136
|
+
raise ValueError(
|
|
137
|
+
"'iss', 'sub', 'aud', 'iat', 'exp', 'nbf' are reserved properties and should not be set as part of 'additional_jwt_payload'"
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
if self._iss:
|
|
141
|
+
payload["iss"] = self._iss.eval(self.config, json_loads=json.loads)
|
|
142
|
+
if self._sub:
|
|
143
|
+
payload["sub"] = self._sub.eval(self.config, json_loads=json.loads)
|
|
144
|
+
if self._aud:
|
|
145
|
+
payload["aud"] = self._aud.eval(self.config, json_loads=json.loads)
|
|
146
|
+
|
|
147
|
+
payload["iat"] = now
|
|
148
|
+
payload["exp"] = exp
|
|
149
|
+
payload["nbf"] = nbf
|
|
150
|
+
return payload
|
|
151
|
+
|
|
152
|
+
def _get_secret_key(self) -> str:
|
|
153
|
+
"""
|
|
154
|
+
Returns the secret key used to sign the JWT.
|
|
155
|
+
"""
|
|
156
|
+
secret_key: str = self._secret_key.eval(self.config, json_loads=json.loads)
|
|
157
|
+
return (
|
|
158
|
+
base64.b64encode(secret_key.encode()).decode()
|
|
159
|
+
if self._base64_encode_secret_key
|
|
160
|
+
else secret_key
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
def _get_signed_token(self) -> Union[str, Any]:
|
|
164
|
+
"""
|
|
165
|
+
Signed the JWT using the provided secret key and algorithm and the generated headers and payload. For additional information on PyJWT see: https://pyjwt.readthedocs.io/en/stable/
|
|
166
|
+
"""
|
|
167
|
+
try:
|
|
168
|
+
return jwt.encode(
|
|
169
|
+
payload=self._get_jwt_payload(),
|
|
170
|
+
key=self._get_secret_key(),
|
|
171
|
+
algorithm=self._algorithm,
|
|
172
|
+
headers=self._get_jwt_headers(),
|
|
173
|
+
)
|
|
174
|
+
except Exception as e:
|
|
175
|
+
raise ValueError(f"Failed to sign token: {e}")
|
|
176
|
+
|
|
177
|
+
def _get_header_prefix(self) -> Union[str, None]:
|
|
178
|
+
"""
|
|
179
|
+
Returns the header prefix to be used when attaching the token to the request.
|
|
180
|
+
"""
|
|
181
|
+
return (
|
|
182
|
+
self._header_prefix.eval(self.config, json_loads=json.loads)
|
|
183
|
+
if self._header_prefix
|
|
184
|
+
else None
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
@property
|
|
188
|
+
def auth_header(self) -> str:
|
|
189
|
+
return "Authorization"
|
|
190
|
+
|
|
191
|
+
@property
|
|
192
|
+
def token(self) -> str:
|
|
193
|
+
return (
|
|
194
|
+
f"{self._get_header_prefix()} {self._get_signed_token()}"
|
|
195
|
+
if self._get_header_prefix()
|
|
196
|
+
else self._get_signed_token()
|
|
197
|
+
)
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from dataclasses import InitVar, dataclass, field
|
|
6
|
+
from datetime import datetime, timedelta
|
|
7
|
+
from typing import Any, List, Mapping, MutableMapping, Optional, Union
|
|
8
|
+
|
|
9
|
+
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
|
|
10
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
|
11
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
|
|
12
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
|
13
|
+
from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
|
|
14
|
+
from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import (
|
|
15
|
+
AbstractOauth2Authenticator,
|
|
16
|
+
)
|
|
17
|
+
from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import (
|
|
18
|
+
SingleUseRefreshTokenOauth2Authenticator,
|
|
19
|
+
)
|
|
20
|
+
from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAuthenticator):
|
|
25
|
+
"""
|
|
26
|
+
Generates OAuth2.0 access tokens from an OAuth2.0 refresh token and client credentials based on
|
|
27
|
+
a declarative connector configuration file. Credentials can be defined explicitly or via interpolation
|
|
28
|
+
at runtime. The generated access token is attached to each request via the Authorization header.
|
|
29
|
+
|
|
30
|
+
Attributes:
|
|
31
|
+
token_refresh_endpoint (Union[InterpolatedString, str]): The endpoint to refresh the access token
|
|
32
|
+
client_id (Union[InterpolatedString, str]): The client id
|
|
33
|
+
client_secret (Union[InterpolatedString, str]): Client secret
|
|
34
|
+
refresh_token (Union[InterpolatedString, str]): The token used to refresh the access token
|
|
35
|
+
access_token_name (Union[InterpolatedString, str]): THe field to extract access token from in the response
|
|
36
|
+
expires_in_name (Union[InterpolatedString, str]): The field to extract expires_in from in the response
|
|
37
|
+
config (Mapping[str, Any]): The user-provided configuration as specified by the source's spec
|
|
38
|
+
scopes (Optional[List[str]]): The scopes to request
|
|
39
|
+
token_expiry_date (Optional[Union[InterpolatedString, str]]): The access token expiration date
|
|
40
|
+
token_expiry_date_format str: format of the datetime; provide it if expires_in is returned in datetime instead of seconds
|
|
41
|
+
token_expiry_is_time_of_expiration bool: set True it if expires_in is returned as time of expiration instead of the number seconds until expiration
|
|
42
|
+
refresh_request_body (Optional[Mapping[str, Any]]): The request body to send in the refresh request
|
|
43
|
+
refresh_request_headers (Optional[Mapping[str, Any]]): The request headers to send in the refresh request
|
|
44
|
+
grant_type: The grant_type to request for access_token. If set to refresh_token, the refresh_token parameter has to be provided
|
|
45
|
+
message_repository (MessageRepository): the message repository used to emit logs on HTTP requests
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
config: Mapping[str, Any]
|
|
49
|
+
parameters: InitVar[Mapping[str, Any]]
|
|
50
|
+
client_id: Optional[Union[InterpolatedString, str]] = None
|
|
51
|
+
client_secret: Optional[Union[InterpolatedString, str]] = None
|
|
52
|
+
token_refresh_endpoint: Optional[Union[InterpolatedString, str]] = None
|
|
53
|
+
refresh_token: Optional[Union[InterpolatedString, str]] = None
|
|
54
|
+
scopes: Optional[List[str]] = None
|
|
55
|
+
token_expiry_date: Optional[Union[InterpolatedString, str]] = None
|
|
56
|
+
_token_expiry_date: Optional[AirbyteDateTime] = field(init=False, repr=False, default=None)
|
|
57
|
+
token_expiry_date_format: Optional[str] = None
|
|
58
|
+
token_expiry_is_time_of_expiration: bool = False
|
|
59
|
+
access_token_name: Union[InterpolatedString, str] = "access_token"
|
|
60
|
+
access_token_value: Optional[Union[InterpolatedString, str]] = None
|
|
61
|
+
client_id_name: Union[InterpolatedString, str] = "client_id"
|
|
62
|
+
client_secret_name: Union[InterpolatedString, str] = "client_secret"
|
|
63
|
+
expires_in_name: Union[InterpolatedString, str] = "expires_in"
|
|
64
|
+
refresh_token_name: Union[InterpolatedString, str] = "refresh_token"
|
|
65
|
+
refresh_request_body: Optional[Mapping[str, Any]] = None
|
|
66
|
+
refresh_request_headers: Optional[Mapping[str, Any]] = None
|
|
67
|
+
grant_type_name: Union[InterpolatedString, str] = "grant_type"
|
|
68
|
+
grant_type: Union[InterpolatedString, str] = "refresh_token"
|
|
69
|
+
message_repository: MessageRepository = NoopMessageRepository()
|
|
70
|
+
profile_assertion: Optional[DeclarativeAuthenticator] = None
|
|
71
|
+
use_profile_assertion: Optional[Union[InterpolatedBoolean, str, bool]] = False
|
|
72
|
+
|
|
73
|
+
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
|
74
|
+
super().__init__()
|
|
75
|
+
if self.token_refresh_endpoint is not None:
|
|
76
|
+
self._token_refresh_endpoint: Optional[InterpolatedString] = InterpolatedString.create(
|
|
77
|
+
self.token_refresh_endpoint, parameters=parameters
|
|
78
|
+
)
|
|
79
|
+
else:
|
|
80
|
+
self._token_refresh_endpoint = None
|
|
81
|
+
self._client_id_name = InterpolatedString.create(self.client_id_name, parameters=parameters)
|
|
82
|
+
self._client_id = (
|
|
83
|
+
InterpolatedString.create(self.client_id, parameters=parameters)
|
|
84
|
+
if self.client_id
|
|
85
|
+
else self.client_id
|
|
86
|
+
)
|
|
87
|
+
self._client_secret_name = InterpolatedString.create(
|
|
88
|
+
self.client_secret_name, parameters=parameters
|
|
89
|
+
)
|
|
90
|
+
self._client_secret = (
|
|
91
|
+
InterpolatedString.create(self.client_secret, parameters=parameters)
|
|
92
|
+
if self.client_secret
|
|
93
|
+
else self.client_secret
|
|
94
|
+
)
|
|
95
|
+
self._refresh_token_name = InterpolatedString.create(
|
|
96
|
+
self.refresh_token_name, parameters=parameters
|
|
97
|
+
)
|
|
98
|
+
if self.refresh_token is not None:
|
|
99
|
+
self._refresh_token: Optional[InterpolatedString] = InterpolatedString.create(
|
|
100
|
+
self.refresh_token, parameters=parameters
|
|
101
|
+
)
|
|
102
|
+
else:
|
|
103
|
+
self._refresh_token = None
|
|
104
|
+
self.access_token_name = InterpolatedString.create(
|
|
105
|
+
self.access_token_name, parameters=parameters
|
|
106
|
+
)
|
|
107
|
+
self.expires_in_name = InterpolatedString.create(
|
|
108
|
+
self.expires_in_name, parameters=parameters
|
|
109
|
+
)
|
|
110
|
+
self.grant_type_name = InterpolatedString.create(
|
|
111
|
+
self.grant_type_name, parameters=parameters
|
|
112
|
+
)
|
|
113
|
+
self.grant_type = InterpolatedString.create(
|
|
114
|
+
"urn:ietf:params:oauth:grant-type:jwt-bearer"
|
|
115
|
+
if self.use_profile_assertion
|
|
116
|
+
else self.grant_type,
|
|
117
|
+
parameters=parameters,
|
|
118
|
+
)
|
|
119
|
+
self._refresh_request_body = InterpolatedMapping(
|
|
120
|
+
self.refresh_request_body or {}, parameters=parameters
|
|
121
|
+
)
|
|
122
|
+
self._refresh_request_headers = InterpolatedMapping(
|
|
123
|
+
self.refresh_request_headers or {}, parameters=parameters
|
|
124
|
+
)
|
|
125
|
+
try:
|
|
126
|
+
if (
|
|
127
|
+
isinstance(self.token_expiry_date, (int, str))
|
|
128
|
+
and str(self.token_expiry_date).isdigit()
|
|
129
|
+
):
|
|
130
|
+
self._token_expiry_date = ab_datetime_parse(self.token_expiry_date)
|
|
131
|
+
else:
|
|
132
|
+
self._token_expiry_date = (
|
|
133
|
+
ab_datetime_parse(
|
|
134
|
+
InterpolatedString.create(
|
|
135
|
+
self.token_expiry_date, parameters=parameters
|
|
136
|
+
).eval(self.config)
|
|
137
|
+
)
|
|
138
|
+
if self.token_expiry_date
|
|
139
|
+
else ab_datetime_now() - timedelta(days=1)
|
|
140
|
+
)
|
|
141
|
+
except ValueError as e:
|
|
142
|
+
raise ValueError(f"Invalid token expiry date format: {e}")
|
|
143
|
+
self.use_profile_assertion = (
|
|
144
|
+
InterpolatedBoolean(self.use_profile_assertion, parameters=parameters)
|
|
145
|
+
if isinstance(self.use_profile_assertion, str)
|
|
146
|
+
else self.use_profile_assertion
|
|
147
|
+
)
|
|
148
|
+
self.assertion_name = "assertion"
|
|
149
|
+
|
|
150
|
+
if self.access_token_value is not None:
|
|
151
|
+
self._access_token_value = InterpolatedString.create(
|
|
152
|
+
self.access_token_value, parameters=parameters
|
|
153
|
+
).eval(self.config)
|
|
154
|
+
else:
|
|
155
|
+
self._access_token_value = None
|
|
156
|
+
|
|
157
|
+
self._access_token: Optional[str] = (
|
|
158
|
+
self._access_token_value if self.access_token_value else None
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
if not self.use_profile_assertion and any(
|
|
162
|
+
client_creds is None for client_creds in [self.client_id, self.client_secret]
|
|
163
|
+
):
|
|
164
|
+
raise ValueError(
|
|
165
|
+
"OAuthAuthenticator configuration error: Both 'client_id' and 'client_secret' are required for the "
|
|
166
|
+
"basic OAuth flow."
|
|
167
|
+
)
|
|
168
|
+
if self.profile_assertion is None and self.use_profile_assertion:
|
|
169
|
+
raise ValueError(
|
|
170
|
+
"OAuthAuthenticator configuration error: 'profile_assertion' is required when using the profile assertion flow."
|
|
171
|
+
)
|
|
172
|
+
if self.get_grant_type() == "refresh_token" and self._refresh_token is None:
|
|
173
|
+
raise ValueError(
|
|
174
|
+
"OAuthAuthenticator configuration error: A 'refresh_token' is required when the 'grant_type' is set to 'refresh_token'."
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
def get_token_refresh_endpoint(self) -> Optional[str]:
|
|
178
|
+
if self._token_refresh_endpoint is not None:
|
|
179
|
+
refresh_token_endpoint: str = self._token_refresh_endpoint.eval(self.config)
|
|
180
|
+
if not refresh_token_endpoint:
|
|
181
|
+
raise ValueError(
|
|
182
|
+
"OAuthAuthenticator was unable to evaluate token_refresh_endpoint parameter"
|
|
183
|
+
)
|
|
184
|
+
return refresh_token_endpoint
|
|
185
|
+
return None
|
|
186
|
+
|
|
187
|
+
def get_client_id_name(self) -> str:
|
|
188
|
+
return self._client_id_name.eval(self.config) # type: ignore # eval returns a string in this context
|
|
189
|
+
|
|
190
|
+
def get_client_id(self) -> str:
|
|
191
|
+
client_id = self._client_id.eval(self.config) if self._client_id else self._client_id
|
|
192
|
+
if not client_id:
|
|
193
|
+
raise ValueError("OAuthAuthenticator was unable to evaluate client_id parameter")
|
|
194
|
+
return client_id # type: ignore # value will be returned as a string, or an error will be raised
|
|
195
|
+
|
|
196
|
+
def get_client_secret_name(self) -> str:
|
|
197
|
+
return self._client_secret_name.eval(self.config) # type: ignore # eval returns a string in this context
|
|
198
|
+
|
|
199
|
+
def get_client_secret(self) -> str:
|
|
200
|
+
client_secret = (
|
|
201
|
+
self._client_secret.eval(self.config) if self._client_secret else self._client_secret
|
|
202
|
+
)
|
|
203
|
+
if not client_secret:
|
|
204
|
+
raise ValueError("OAuthAuthenticator was unable to evaluate client_secret parameter")
|
|
205
|
+
return client_secret # type: ignore # value will be returned as a string, or an error will be raised
|
|
206
|
+
|
|
207
|
+
def get_refresh_token_name(self) -> str:
|
|
208
|
+
return self._refresh_token_name.eval(self.config) # type: ignore # eval returns a string in this context
|
|
209
|
+
|
|
210
|
+
def get_refresh_token(self) -> Optional[str]:
|
|
211
|
+
return None if self._refresh_token is None else str(self._refresh_token.eval(self.config))
|
|
212
|
+
|
|
213
|
+
def get_scopes(self) -> List[str]:
|
|
214
|
+
return self.scopes or []
|
|
215
|
+
|
|
216
|
+
def get_access_token_name(self) -> str:
|
|
217
|
+
return self.access_token_name.eval(self.config) # type: ignore # eval returns a string in this context
|
|
218
|
+
|
|
219
|
+
def get_expires_in_name(self) -> str:
|
|
220
|
+
return self.expires_in_name.eval(self.config) # type: ignore # eval returns a string in this context
|
|
221
|
+
|
|
222
|
+
def get_grant_type_name(self) -> str:
|
|
223
|
+
return self.grant_type_name.eval(self.config) # type: ignore # eval returns a string in this context
|
|
224
|
+
|
|
225
|
+
def get_grant_type(self) -> str:
|
|
226
|
+
return self.grant_type.eval(self.config) # type: ignore # eval returns a string in this context
|
|
227
|
+
|
|
228
|
+
def get_refresh_request_body(self) -> Mapping[str, Any]:
|
|
229
|
+
return self._refresh_request_body.eval(self.config)
|
|
230
|
+
|
|
231
|
+
def get_refresh_request_headers(self) -> Mapping[str, Any]:
|
|
232
|
+
return self._refresh_request_headers.eval(self.config)
|
|
233
|
+
|
|
234
|
+
def get_token_expiry_date(self) -> AirbyteDateTime:
|
|
235
|
+
if not self._has_access_token_been_initialized():
|
|
236
|
+
return AirbyteDateTime.from_datetime(datetime.min)
|
|
237
|
+
return self._token_expiry_date # type: ignore # _token_expiry_date is an AirbyteDateTime. It is never None despite what mypy thinks
|
|
238
|
+
|
|
239
|
+
def _has_access_token_been_initialized(self) -> bool:
|
|
240
|
+
return self._access_token is not None
|
|
241
|
+
|
|
242
|
+
def set_token_expiry_date(self, value: Union[str, int]) -> None:
|
|
243
|
+
self._token_expiry_date = self._parse_token_expiration_date(value)
|
|
244
|
+
|
|
245
|
+
def get_assertion_name(self) -> str:
|
|
246
|
+
return self.assertion_name
|
|
247
|
+
|
|
248
|
+
def get_assertion(self) -> str:
|
|
249
|
+
if self.profile_assertion is None:
|
|
250
|
+
raise ValueError("profile_assertion is not set")
|
|
251
|
+
return self.profile_assertion.token
|
|
252
|
+
|
|
253
|
+
def build_refresh_request_body(self) -> Mapping[str, Any]:
|
|
254
|
+
"""
|
|
255
|
+
Returns the request body to set on the refresh request
|
|
256
|
+
|
|
257
|
+
Override to define additional parameters
|
|
258
|
+
"""
|
|
259
|
+
if self.use_profile_assertion:
|
|
260
|
+
return {
|
|
261
|
+
self.get_grant_type_name(): self.get_grant_type(),
|
|
262
|
+
self.get_assertion_name(): self.get_assertion(),
|
|
263
|
+
}
|
|
264
|
+
return super().build_refresh_request_body()
|
|
265
|
+
|
|
266
|
+
@property
|
|
267
|
+
def access_token(self) -> str:
|
|
268
|
+
if self._access_token is None:
|
|
269
|
+
raise ValueError("access_token is not set")
|
|
270
|
+
return self._access_token
|
|
271
|
+
|
|
272
|
+
@access_token.setter
|
|
273
|
+
def access_token(self, value: str) -> None:
|
|
274
|
+
self._access_token = value
|
|
275
|
+
|
|
276
|
+
@property
|
|
277
|
+
def _message_repository(self) -> MessageRepository:
|
|
278
|
+
"""
|
|
279
|
+
Overriding AbstractOauth2Authenticator._message_repository to allow for HTTP request logs
|
|
280
|
+
"""
|
|
281
|
+
return self.message_repository
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
@dataclass
|
|
285
|
+
class DeclarativeSingleUseRefreshTokenOauth2Authenticator(
|
|
286
|
+
SingleUseRefreshTokenOauth2Authenticator, DeclarativeAuthenticator
|
|
287
|
+
):
|
|
288
|
+
"""
|
|
289
|
+
Declarative version of SingleUseRefreshTokenOauth2Authenticator which can be used in declarative connectors.
|
|
290
|
+
"""
|
|
291
|
+
|
|
292
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
293
|
+
super().__init__(*args, **kwargs)
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Any, List, Mapping
|
|
7
|
+
|
|
8
|
+
import dpath
|
|
9
|
+
|
|
10
|
+
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class SelectiveAuthenticator(DeclarativeAuthenticator):
|
|
15
|
+
"""Authenticator that selects concrete implementation based on specific config value."""
|
|
16
|
+
|
|
17
|
+
config: Mapping[str, Any]
|
|
18
|
+
authenticators: Mapping[str, DeclarativeAuthenticator]
|
|
19
|
+
authenticator_selection_path: List[str]
|
|
20
|
+
|
|
21
|
+
# returns "DeclarativeAuthenticator", but must return a subtype of "SelectiveAuthenticator"
|
|
22
|
+
def __new__( # type: ignore[misc]
|
|
23
|
+
cls,
|
|
24
|
+
config: Mapping[str, Any],
|
|
25
|
+
authenticators: Mapping[str, DeclarativeAuthenticator],
|
|
26
|
+
authenticator_selection_path: List[str],
|
|
27
|
+
*arg: Any,
|
|
28
|
+
**kwargs: Any,
|
|
29
|
+
) -> DeclarativeAuthenticator:
|
|
30
|
+
try:
|
|
31
|
+
selected_key = str(
|
|
32
|
+
dpath.get(
|
|
33
|
+
config, # type: ignore[arg-type] # Dpath wants mutable mapping but doesn't need it.
|
|
34
|
+
authenticator_selection_path,
|
|
35
|
+
)
|
|
36
|
+
)
|
|
37
|
+
except KeyError as err:
|
|
38
|
+
raise ValueError(
|
|
39
|
+
"The path from `authenticator_selection_path` is not found in the config."
|
|
40
|
+
) from err
|
|
41
|
+
|
|
42
|
+
try:
|
|
43
|
+
return authenticators[selected_key]
|
|
44
|
+
except KeyError as err:
|
|
45
|
+
raise ValueError(f"The authenticator `{selected_key}` is not found.") from err
|