airbyte-cdk 0.72.1__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- airbyte_cdk/__init__.py +355 -6
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +230 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +29 -10
- airbyte_cdk/connector.py +24 -24
- airbyte_cdk/connector_builder/README.md +53 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +37 -11
- airbyte_cdk/connector_builder/main.py +45 -13
- airbyte_cdk/connector_builder/message_grouper.py +189 -50
- airbyte_cdk/connector_builder/models.py +3 -2
- airbyte_cdk/destinations/__init__.py +4 -3
- airbyte_cdk/destinations/destination.py +54 -20
- airbyte_cdk/destinations/vector_db_based/README.md +37 -0
- airbyte_cdk/destinations/vector_db_based/config.py +40 -17
- airbyte_cdk/destinations/vector_db_based/document_processor.py +56 -17
- airbyte_cdk/destinations/vector_db_based/embedder.py +57 -15
- airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
- airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +24 -5
- airbyte_cdk/entrypoint.py +153 -44
- airbyte_cdk/exception_handler.py +21 -3
- airbyte_cdk/logger.py +30 -44
- airbyte_cdk/models/__init__.py +13 -2
- airbyte_cdk/models/airbyte_protocol.py +86 -1
- airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
- airbyte_cdk/models/file_transfer_record_message.py +13 -0
- airbyte_cdk/models/well_known_types.py +1 -1
- airbyte_cdk/sources/__init__.py +5 -1
- airbyte_cdk/sources/abstract_source.py +125 -79
- airbyte_cdk/sources/concurrent_source/__init__.py +7 -2
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +102 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +29 -36
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +94 -10
- airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +20 -14
- airbyte_cdk/sources/config.py +3 -2
- airbyte_cdk/sources/connector_state_manager.py +49 -83
- airbyte_cdk/sources/declarative/async_job/job.py +52 -0
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +497 -0
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +75 -0
- airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
- airbyte_cdk/sources/declarative/async_job/status.py +24 -0
- airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
- airbyte_cdk/sources/declarative/auth/__init__.py +2 -3
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +191 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +60 -20
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +10 -2
- airbyte_cdk/sources/declarative/auth/token.py +28 -10
- airbyte_cdk/sources/declarative/auth/token_provider.py +9 -8
- airbyte_cdk/sources/declarative/checks/check_stream.py +16 -8
- airbyte_cdk/sources/declarative/checks/connection_checker.py +4 -2
- airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +421 -0
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +4 -0
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +26 -6
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +1185 -85
- airbyte_cdk/sources/declarative/declarative_source.py +5 -2
- airbyte_cdk/sources/declarative/declarative_stream.py +95 -9
- airbyte_cdk/sources/declarative/decoders/__init__.py +23 -2
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +97 -0
- airbyte_cdk/sources/declarative/decoders/decoder.py +11 -4
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +92 -5
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
- airbyte_cdk/sources/declarative/extractors/__init__.py +12 -1
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +29 -24
- airbyte_cdk/sources/declarative/extractors/http_selector.py +4 -5
- airbyte_cdk/sources/declarative/extractors/record_extractor.py +2 -3
- airbyte_cdk/sources/declarative/extractors/record_filter.py +65 -8
- airbyte_cdk/sources/declarative/extractors/record_selector.py +85 -26
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +177 -0
- airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
- airbyte_cdk/sources/declarative/incremental/__init__.py +25 -3
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +156 -48
- airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +350 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +159 -74
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
- airbyte_cdk/sources/declarative/interpolation/filters.py +27 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +23 -5
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +12 -8
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +13 -6
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +21 -6
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +9 -3
- airbyte_cdk/sources/declarative/interpolation/jinja.py +72 -37
- airbyte_cdk/sources/declarative/interpolation/macros.py +72 -17
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +193 -52
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
- airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
- airbyte_cdk/sources/declarative/models/__init__.py +1 -1
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +1319 -603
- airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +2 -2
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +26 -4
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +26 -15
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +1695 -225
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +24 -4
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +39 -9
- airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
- airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +15 -3
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +222 -39
- airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +19 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +19 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +4 -2
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +41 -9
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +29 -14
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +5 -13
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +32 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +46 -56
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +6 -32
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +119 -41
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +228 -0
- airbyte_cdk/sources/declarative/requesters/http_requester.py +98 -344
- airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +14 -3
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +105 -46
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +14 -8
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +19 -8
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +9 -3
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +53 -21
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +42 -19
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +25 -12
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +13 -10
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +26 -13
- airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +15 -2
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +91 -0
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +31 -14
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +27 -15
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +63 -10
- airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +1 -1
- airbyte_cdk/sources/declarative/requesters/requester.py +9 -17
- airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
- airbyte_cdk/sources/declarative/retrievers/__init__.py +6 -2
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +100 -0
- airbyte_cdk/sources/declarative/retrievers/retriever.py +1 -3
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +228 -72
- airbyte_cdk/sources/declarative/schema/__init__.py +14 -1
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +5 -3
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +236 -0
- airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +8 -8
- airbyte_cdk/sources/declarative/spec/spec.py +12 -5
- airbyte_cdk/sources/declarative/stream_slicers/__init__.py +1 -2
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +88 -0
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +9 -14
- airbyte_cdk/sources/declarative/transformations/add_fields.py +19 -11
- airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
- airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +13 -10
- airbyte_cdk/sources/declarative/transformations/transformation.py +5 -5
- airbyte_cdk/sources/declarative/types.py +19 -110
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +31 -10
- airbyte_cdk/sources/embedded/base_integration.py +16 -5
- airbyte_cdk/sources/embedded/catalog.py +16 -4
- airbyte_cdk/sources/embedded/runner.py +19 -3
- airbyte_cdk/sources/embedded/tools.py +5 -2
- airbyte_cdk/sources/file_based/README.md +152 -0
- airbyte_cdk/sources/file_based/__init__.py +24 -0
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +9 -2
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +22 -6
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +46 -10
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +58 -10
- airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
- airbyte_cdk/sources/file_based/config/csv_format.py +29 -10
- airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +16 -4
- airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
- airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
- airbyte_cdk/sources/file_based/config/unstructured_format.py +13 -5
- airbyte_cdk/sources/file_based/discovery_policy/__init__.py +6 -2
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
- airbyte_cdk/sources/file_based/exceptions.py +52 -15
- airbyte_cdk/sources/file_based/file_based_source.py +163 -33
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +83 -5
- airbyte_cdk/sources/file_based/file_types/__init__.py +14 -1
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +75 -24
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +116 -34
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +37 -0
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +24 -8
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +60 -18
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +145 -41
- airbyte_cdk/sources/file_based/remote_file.py +1 -1
- airbyte_cdk/sources/file_based/schema_helpers.py +38 -10
- airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +50 -13
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +67 -27
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +5 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +14 -23
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +54 -18
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +21 -9
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +27 -10
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +175 -45
- airbyte_cdk/sources/http_logger.py +8 -3
- airbyte_cdk/sources/message/__init__.py +7 -1
- airbyte_cdk/sources/message/repository.py +18 -4
- airbyte_cdk/sources/source.py +42 -38
- airbyte_cdk/sources/streams/__init__.py +2 -2
- airbyte_cdk/sources/streams/availability_strategy.py +54 -3
- airbyte_cdk/sources/streams/call_rate.py +64 -21
- airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
- airbyte_cdk/sources/{declarative/incremental → streams/checkpoint}/cursor.py +17 -14
- airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
- airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
- airbyte_cdk/sources/streams/concurrent/README.md +7 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +7 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +84 -75
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +30 -2
- airbyte_cdk/sources/streams/concurrent/cursor.py +298 -42
- airbyte_cdk/sources/streams/concurrent/default_stream.py +12 -3
- airbyte_cdk/sources/streams/concurrent/exceptions.py +3 -0
- airbyte_cdk/sources/streams/concurrent/helpers.py +14 -3
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +12 -3
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +10 -3
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -16
- airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +15 -5
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +109 -17
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +90 -72
- airbyte_cdk/sources/streams/core.py +412 -87
- airbyte_cdk/sources/streams/http/__init__.py +2 -1
- airbyte_cdk/sources/streams/http/availability_strategy.py +12 -101
- airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
- airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
- airbyte_cdk/sources/streams/http/exceptions.py +27 -7
- airbyte_cdk/sources/streams/http/http.py +369 -246
- airbyte_cdk/sources/streams/http/http_client.py +531 -0
- airbyte_cdk/sources/streams/http/rate_limiting.py +76 -12
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +28 -9
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +90 -35
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
- airbyte_cdk/sources/types.py +154 -0
- airbyte_cdk/sources/utils/record_helper.py +36 -21
- airbyte_cdk/sources/utils/schema_helpers.py +13 -6
- airbyte_cdk/sources/utils/slice_logger.py +4 -1
- airbyte_cdk/sources/utils/transform.py +54 -20
- airbyte_cdk/sql/_util/hashing.py +34 -0
- airbyte_cdk/sql/_util/name_normalizers.py +92 -0
- airbyte_cdk/sql/constants.py +32 -0
- airbyte_cdk/sql/exceptions.py +235 -0
- airbyte_cdk/sql/secrets.py +123 -0
- airbyte_cdk/sql/shared/__init__.py +15 -0
- airbyte_cdk/sql/shared/catalog_providers.py +145 -0
- airbyte_cdk/sql/shared/sql_processor.py +786 -0
- airbyte_cdk/sql/types.py +160 -0
- airbyte_cdk/test/catalog_builder.py +70 -18
- airbyte_cdk/test/entrypoint_wrapper.py +117 -42
- airbyte_cdk/test/mock_http/__init__.py +1 -1
- airbyte_cdk/test/mock_http/matcher.py +6 -0
- airbyte_cdk/test/mock_http/mocker.py +57 -10
- airbyte_cdk/test/mock_http/request.py +19 -3
- airbyte_cdk/test/mock_http/response.py +3 -1
- airbyte_cdk/test/mock_http/response_builder.py +32 -16
- airbyte_cdk/test/state_builder.py +18 -10
- airbyte_cdk/test/utils/__init__.py +1 -0
- airbyte_cdk/test/utils/data.py +24 -0
- airbyte_cdk/test/utils/http_mocking.py +16 -0
- airbyte_cdk/test/utils/manifest_only_fixtures.py +60 -0
- airbyte_cdk/test/utils/reading.py +26 -0
- airbyte_cdk/utils/__init__.py +2 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +5 -3
- airbyte_cdk/utils/analytics_message.py +10 -2
- airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
- airbyte_cdk/utils/event_timing.py +10 -10
- airbyte_cdk/utils/mapping_helpers.py +3 -1
- airbyte_cdk/utils/message_utils.py +20 -11
- airbyte_cdk/utils/print_buffer.py +75 -0
- airbyte_cdk/utils/schema_inferrer.py +198 -28
- airbyte_cdk/utils/slice_hasher.py +30 -0
- airbyte_cdk/utils/spec_schema_transformations.py +6 -3
- airbyte_cdk/utils/stream_status_utils.py +8 -1
- airbyte_cdk/utils/traced_exception.py +61 -21
- airbyte_cdk-6.13.1.dev4106.dist-info/METADATA +109 -0
- airbyte_cdk-6.13.1.dev4106.dist-info/RECORD +349 -0
- {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.13.1.dev4106.dist-info}/WHEEL +1 -2
- airbyte_cdk-6.13.1.dev4106.dist-info/entry_points.txt +3 -0
- airbyte_cdk/sources/declarative/create_partial.py +0 -92
- airbyte_cdk/sources/declarative/parsers/class_types_registry.py +0 -102
- airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +0 -64
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_action.py +0 -16
- airbyte_cdk/sources/declarative/requesters/error_handlers/response_status.py +0 -68
- airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +0 -114
- airbyte_cdk/sources/deprecated/base_source.py +0 -94
- airbyte_cdk/sources/deprecated/client.py +0 -99
- airbyte_cdk/sources/singer/__init__.py +0 -8
- airbyte_cdk/sources/singer/singer_helpers.py +0 -304
- airbyte_cdk/sources/singer/source.py +0 -186
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -23
- airbyte_cdk/sources/streams/http/auth/__init__.py +0 -17
- airbyte_cdk/sources/streams/http/auth/core.py +0 -29
- airbyte_cdk/sources/streams/http/auth/oauth.py +0 -113
- airbyte_cdk/sources/streams/http/auth/token.py +0 -47
- airbyte_cdk/sources/streams/utils/stream_helper.py +0 -40
- airbyte_cdk/sources/utils/catalog_helpers.py +0 -22
- airbyte_cdk/sources/utils/schema_models.py +0 -84
- airbyte_cdk-0.72.1.dist-info/METADATA +0 -243
- airbyte_cdk-0.72.1.dist-info/RECORD +0 -466
- airbyte_cdk-0.72.1.dist-info/top_level.txt +0 -3
- source_declarative_manifest/main.py +0 -29
- unit_tests/connector_builder/__init__.py +0 -3
- unit_tests/connector_builder/test_connector_builder_handler.py +0 -871
- unit_tests/connector_builder/test_message_grouper.py +0 -713
- unit_tests/connector_builder/utils.py +0 -27
- unit_tests/destinations/test_destination.py +0 -243
- unit_tests/singer/test_singer_helpers.py +0 -56
- unit_tests/singer/test_singer_source.py +0 -112
- unit_tests/sources/__init__.py +0 -0
- unit_tests/sources/concurrent_source/__init__.py +0 -3
- unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +0 -106
- unit_tests/sources/declarative/__init__.py +0 -3
- unit_tests/sources/declarative/auth/__init__.py +0 -3
- unit_tests/sources/declarative/auth/test_oauth.py +0 -331
- unit_tests/sources/declarative/auth/test_selective_authenticator.py +0 -39
- unit_tests/sources/declarative/auth/test_session_token_auth.py +0 -182
- unit_tests/sources/declarative/auth/test_token_auth.py +0 -200
- unit_tests/sources/declarative/auth/test_token_provider.py +0 -73
- unit_tests/sources/declarative/checks/__init__.py +0 -3
- unit_tests/sources/declarative/checks/test_check_stream.py +0 -146
- unit_tests/sources/declarative/decoders/__init__.py +0 -0
- unit_tests/sources/declarative/decoders/test_json_decoder.py +0 -16
- unit_tests/sources/declarative/external_component.py +0 -13
- unit_tests/sources/declarative/extractors/__init__.py +0 -3
- unit_tests/sources/declarative/extractors/test_dpath_extractor.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_filter.py +0 -55
- unit_tests/sources/declarative/extractors/test_record_selector.py +0 -179
- unit_tests/sources/declarative/incremental/__init__.py +0 -0
- unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +0 -860
- unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +0 -406
- unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +0 -332
- unit_tests/sources/declarative/interpolation/__init__.py +0 -3
- unit_tests/sources/declarative/interpolation/test_filters.py +0 -80
- unit_tests/sources/declarative/interpolation/test_interpolated_boolean.py +0 -40
- unit_tests/sources/declarative/interpolation/test_interpolated_mapping.py +0 -35
- unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +0 -45
- unit_tests/sources/declarative/interpolation/test_interpolated_string.py +0 -25
- unit_tests/sources/declarative/interpolation/test_jinja.py +0 -240
- unit_tests/sources/declarative/interpolation/test_macros.py +0 -73
- unit_tests/sources/declarative/parsers/__init__.py +0 -3
- unit_tests/sources/declarative/parsers/test_manifest_component_transformer.py +0 -406
- unit_tests/sources/declarative/parsers/test_manifest_reference_resolver.py +0 -139
- unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +0 -1847
- unit_tests/sources/declarative/parsers/testing_components.py +0 -36
- unit_tests/sources/declarative/partition_routers/__init__.py +0 -3
- unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +0 -155
- unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +0 -14
- unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +0 -404
- unit_tests/sources/declarative/requesters/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_constant_backoff.py +0 -34
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_exponential_backoff.py +0 -36
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_header_helper.py +0 -38
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +0 -35
- unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_until_time_from_header.py +0 -64
- unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +0 -213
- unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +0 -178
- unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +0 -121
- unit_tests/sources/declarative/requesters/error_handlers/test_response_status.py +0 -44
- unit_tests/sources/declarative/requesters/paginators/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +0 -64
- unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +0 -313
- unit_tests/sources/declarative/requesters/paginators/test_no_paginator.py +0 -12
- unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +0 -58
- unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +0 -70
- unit_tests/sources/declarative/requesters/paginators/test_request_option.py +0 -43
- unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +0 -105
- unit_tests/sources/declarative/requesters/request_options/__init__.py +0 -3
- unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +0 -101
- unit_tests/sources/declarative/requesters/test_http_requester.py +0 -974
- unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py +0 -32
- unit_tests/sources/declarative/retrievers/__init__.py +0 -3
- unit_tests/sources/declarative/retrievers/test_simple_retriever.py +0 -542
- unit_tests/sources/declarative/schema/__init__.py +0 -6
- unit_tests/sources/declarative/schema/source_test/SourceTest.py +0 -8
- unit_tests/sources/declarative/schema/source_test/__init__.py +0 -3
- unit_tests/sources/declarative/schema/test_default_schema_loader.py +0 -32
- unit_tests/sources/declarative/schema/test_inline_schema_loader.py +0 -19
- unit_tests/sources/declarative/schema/test_json_file_schema_loader.py +0 -26
- unit_tests/sources/declarative/states/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/__init__.py +0 -3
- unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +0 -225
- unit_tests/sources/declarative/test_create_partial.py +0 -83
- unit_tests/sources/declarative/test_declarative_stream.py +0 -103
- unit_tests/sources/declarative/test_manifest_declarative_source.py +0 -1260
- unit_tests/sources/declarative/test_types.py +0 -39
- unit_tests/sources/declarative/test_yaml_declarative_source.py +0 -148
- unit_tests/sources/file_based/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/__init__.py +0 -0
- unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +0 -100
- unit_tests/sources/file_based/config/__init__.py +0 -0
- unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +0 -28
- unit_tests/sources/file_based/config/test_csv_format.py +0 -34
- unit_tests/sources/file_based/config/test_file_based_stream_config.py +0 -84
- unit_tests/sources/file_based/discovery_policy/__init__.py +0 -0
- unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py +0 -31
- unit_tests/sources/file_based/file_types/__init__.py +0 -0
- unit_tests/sources/file_based/file_types/test_avro_parser.py +0 -243
- unit_tests/sources/file_based/file_types/test_csv_parser.py +0 -546
- unit_tests/sources/file_based/file_types/test_jsonl_parser.py +0 -158
- unit_tests/sources/file_based/file_types/test_parquet_parser.py +0 -274
- unit_tests/sources/file_based/file_types/test_unstructured_parser.py +0 -593
- unit_tests/sources/file_based/helpers.py +0 -70
- unit_tests/sources/file_based/in_memory_files_source.py +0 -211
- unit_tests/sources/file_based/scenarios/__init__.py +0 -0
- unit_tests/sources/file_based/scenarios/avro_scenarios.py +0 -744
- unit_tests/sources/file_based/scenarios/check_scenarios.py +0 -220
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +0 -2844
- unit_tests/sources/file_based/scenarios/csv_scenarios.py +0 -3105
- unit_tests/sources/file_based/scenarios/file_based_source_builder.py +0 -91
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +0 -1926
- unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +0 -930
- unit_tests/sources/file_based/scenarios/parquet_scenarios.py +0 -754
- unit_tests/sources/file_based/scenarios/scenario_builder.py +0 -234
- unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +0 -608
- unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +0 -746
- unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +0 -726
- unit_tests/sources/file_based/stream/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/__init__.py +0 -0
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +0 -362
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +0 -458
- unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +0 -310
- unit_tests/sources/file_based/stream/test_default_file_based_stream.py +0 -244
- unit_tests/sources/file_based/test_file_based_scenarios.py +0 -320
- unit_tests/sources/file_based/test_file_based_stream_reader.py +0 -272
- unit_tests/sources/file_based/test_scenarios.py +0 -253
- unit_tests/sources/file_based/test_schema_helpers.py +0 -346
- unit_tests/sources/fixtures/__init__.py +0 -3
- unit_tests/sources/fixtures/source_test_fixture.py +0 -153
- unit_tests/sources/message/__init__.py +0 -0
- unit_tests/sources/message/test_repository.py +0 -153
- unit_tests/sources/streams/__init__.py +0 -0
- unit_tests/sources/streams/concurrent/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/__init__.py +0 -3
- unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +0 -250
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +0 -140
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +0 -452
- unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py +0 -76
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py +0 -418
- unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +0 -142
- unit_tests/sources/streams/concurrent/scenarios/utils.py +0 -55
- unit_tests/sources/streams/concurrent/test_adapters.py +0 -380
- unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +0 -684
- unit_tests/sources/streams/concurrent/test_cursor.py +0 -139
- unit_tests/sources/streams/concurrent/test_datetime_state_converter.py +0 -369
- unit_tests/sources/streams/concurrent/test_default_stream.py +0 -197
- unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +0 -90
- unit_tests/sources/streams/concurrent/test_partition_reader.py +0 -67
- unit_tests/sources/streams/concurrent/test_thread_pool_manager.py +0 -106
- unit_tests/sources/streams/http/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/__init__.py +0 -0
- unit_tests/sources/streams/http/auth/test_auth.py +0 -173
- unit_tests/sources/streams/http/requests_native_auth/__init__.py +0 -0
- unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +0 -423
- unit_tests/sources/streams/http/test_availability_strategy.py +0 -180
- unit_tests/sources/streams/http/test_http.py +0 -635
- unit_tests/sources/streams/test_availability_strategy.py +0 -70
- unit_tests/sources/streams/test_call_rate.py +0 -300
- unit_tests/sources/streams/test_stream_read.py +0 -405
- unit_tests/sources/streams/test_streams_core.py +0 -184
- unit_tests/sources/test_abstract_source.py +0 -1442
- unit_tests/sources/test_concurrent_source.py +0 -112
- unit_tests/sources/test_config.py +0 -92
- unit_tests/sources/test_connector_state_manager.py +0 -482
- unit_tests/sources/test_http_logger.py +0 -252
- unit_tests/sources/test_integration_source.py +0 -86
- unit_tests/sources/test_source.py +0 -684
- unit_tests/sources/test_source_read.py +0 -460
- unit_tests/test/__init__.py +0 -0
- unit_tests/test/mock_http/__init__.py +0 -0
- unit_tests/test/mock_http/test_matcher.py +0 -53
- unit_tests/test/mock_http/test_mocker.py +0 -214
- unit_tests/test/mock_http/test_request.py +0 -117
- unit_tests/test/mock_http/test_response_builder.py +0 -177
- unit_tests/test/test_entrypoint_wrapper.py +0 -240
- unit_tests/utils/__init__.py +0 -0
- unit_tests/utils/test_datetime_format_inferrer.py +0 -60
- unit_tests/utils/test_mapping_helpers.py +0 -54
- unit_tests/utils/test_message_utils.py +0 -91
- unit_tests/utils/test_rate_limiting.py +0 -26
- unit_tests/utils/test_schema_inferrer.py +0 -202
- unit_tests/utils/test_secret_utils.py +0 -135
- unit_tests/utils/test_stream_status_utils.py +0 -61
- unit_tests/utils/test_traced_exception.py +0 -107
- /airbyte_cdk/sources/{deprecated → declarative/async_job}/__init__.py +0 -0
- {source_declarative_manifest → airbyte_cdk/sources/declarative/migrations}/__init__.py +0 -0
- {unit_tests/destinations → airbyte_cdk/sql}/__init__.py +0 -0
- {unit_tests/singer → airbyte_cdk/sql/_util}/__init__.py +0 -0
- {airbyte_cdk-0.72.1.dist-info → airbyte_cdk-6.13.1.dev4106.dist-info}/LICENSE.txt +0 -0
airbyte_cdk/__init__.py
CHANGED
@@ -1,9 +1,358 @@
|
|
1
|
-
#
|
2
1
|
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
|
3
|
-
|
2
|
+
"""
|
3
|
+
# Welcome to the Airbyte Python CDK!
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
from .logger import AirbyteLogger
|
5
|
+
The Airbyte Python CDK is a Python library that provides a set of tools to help you build
|
6
|
+
connectors for the Airbyte platform.
|
8
7
|
|
9
|
-
|
8
|
+
## Building Source Connectors
|
9
|
+
|
10
|
+
To build a source connector, you will want to refer to
|
11
|
+
the following classes and modules:
|
12
|
+
|
13
|
+
- `airbyte_cdk.sources`
|
14
|
+
- `airbyte_cdk.sources.concurrent_source`
|
15
|
+
- `airbyte_cdk.sources.config`
|
16
|
+
- `airbyte_cdk.sources.file_based`
|
17
|
+
- `airbyte_cdk.sources.streams`
|
18
|
+
|
19
|
+
## Building Destination Connectors
|
20
|
+
|
21
|
+
To build a destination connector, you will want to refer to
|
22
|
+
the following classes and modules:
|
23
|
+
|
24
|
+
- `airbyte_cdk.destinations`
|
25
|
+
- `airbyte_cdk.destinations.Destination`
|
26
|
+
- `airbyte_cdk.destinations.vector_db_based`
|
27
|
+
|
28
|
+
## Working with Airbyte Protocol Models
|
29
|
+
|
30
|
+
The Airbyte CDK provides a set of classes that help you work with the Airbyte protocol models:
|
31
|
+
|
32
|
+
- `airbyte_cdk.models.airbyte_protocol`
|
33
|
+
- `airbyte_cdk.models.airbyte_protocol_serializers`
|
34
|
+
|
35
|
+
---
|
36
|
+
|
37
|
+
API Reference
|
38
|
+
|
39
|
+
---
|
40
|
+
|
41
|
+
"""
|
42
|
+
|
43
|
+
# Warning: The below imports are not stable and will cause circular
|
44
|
+
# dependencies if auto-sorted with isort. Please keep them in the same order.
|
45
|
+
# TODO: Submodules should import from lower-level modules, rather than importing from here.
|
46
|
+
# Imports should also be placed in `if TYPE_CHECKING` blocks if they are only used as type
|
47
|
+
# hints - again, to avoid circular dependencies.
|
48
|
+
# Once those issues are resolved, the below can be sorted with isort.
|
49
|
+
import dunamai as _dunamai
|
50
|
+
|
51
|
+
from .config_observation import (
|
52
|
+
create_connector_config_control_message,
|
53
|
+
emit_configuration_as_airbyte_control_message,
|
54
|
+
)
|
55
|
+
from .connector import BaseConnector, Connector
|
56
|
+
from .destinations import Destination
|
57
|
+
from .entrypoint import AirbyteEntrypoint, launch
|
58
|
+
from .logger import AirbyteLogFormatter, init_logger
|
59
|
+
from .models import (
|
60
|
+
AdvancedAuth,
|
61
|
+
AirbyteConnectionStatus,
|
62
|
+
AirbyteLogMessage,
|
63
|
+
AirbyteMessage,
|
64
|
+
AirbyteRecordMessage,
|
65
|
+
AirbyteStream,
|
66
|
+
ConfiguredAirbyteCatalog,
|
67
|
+
ConfiguredAirbyteStream,
|
68
|
+
ConnectorSpecification,
|
69
|
+
DestinationSyncMode,
|
70
|
+
FailureType,
|
71
|
+
Level,
|
72
|
+
OAuthConfigSpecification,
|
73
|
+
OrchestratorType,
|
74
|
+
Status,
|
75
|
+
SyncMode,
|
76
|
+
Type,
|
77
|
+
)
|
78
|
+
from .sources import AbstractSource, Source
|
79
|
+
from .sources.concurrent_source.concurrent_source import ConcurrentSource
|
80
|
+
from .sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
|
81
|
+
from .sources.config import BaseConfig
|
82
|
+
from .sources.connector_state_manager import ConnectorStateManager
|
83
|
+
from .sources.declarative.auth import DeclarativeOauth2Authenticator
|
84
|
+
from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth
|
85
|
+
from .sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator
|
86
|
+
from .sources.declarative.auth.token import (
|
87
|
+
ApiKeyAuthenticator,
|
88
|
+
BasicHttpAuthenticator,
|
89
|
+
BearerAuthenticator,
|
90
|
+
)
|
91
|
+
from .sources.declarative.datetime.min_max_datetime import MinMaxDatetime
|
92
|
+
from .sources.declarative.declarative_stream import DeclarativeStream
|
93
|
+
from .sources.declarative.decoders import Decoder, JsonDecoder
|
94
|
+
from .sources.declarative.exceptions import ReadException
|
95
|
+
from .sources.declarative.extractors import DpathExtractor, RecordSelector
|
96
|
+
from .sources.declarative.extractors.record_extractor import RecordExtractor
|
97
|
+
from .sources.declarative.extractors.record_filter import RecordFilter
|
98
|
+
from .sources.declarative.incremental import DatetimeBasedCursor
|
99
|
+
from .sources.declarative.interpolation import InterpolatedBoolean, InterpolatedString
|
100
|
+
from .sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
|
101
|
+
from .sources.declarative.migrations.legacy_to_per_partition_state_migration import (
|
102
|
+
LegacyToPerPartitionStateMigration,
|
103
|
+
)
|
104
|
+
from .sources.declarative.partition_routers import (
|
105
|
+
CartesianProductStreamSlicer,
|
106
|
+
SinglePartitionRouter,
|
107
|
+
SubstreamPartitionRouter,
|
108
|
+
)
|
109
|
+
from .sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig
|
110
|
+
from .sources.declarative.requesters import HttpRequester, Requester
|
111
|
+
from .sources.declarative.requesters.error_handlers import BackoffStrategy
|
112
|
+
from .sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy
|
113
|
+
from .sources.declarative.requesters.paginators.strategies import (
|
114
|
+
CursorPaginationStrategy,
|
115
|
+
OffsetIncrement,
|
116
|
+
PageIncrement,
|
117
|
+
StopConditionPaginationStrategyDecorator,
|
118
|
+
)
|
119
|
+
from .sources.declarative.requesters.request_option import RequestOption, RequestOptionType
|
120
|
+
from .sources.declarative.requesters.request_options.default_request_options_provider import (
|
121
|
+
DefaultRequestOptionsProvider,
|
122
|
+
)
|
123
|
+
from .sources.declarative.requesters.request_options.interpolated_request_input_provider import (
|
124
|
+
InterpolatedRequestInputProvider,
|
125
|
+
)
|
126
|
+
from .sources.declarative.requesters.requester import HttpMethod
|
127
|
+
from .sources.declarative.retrievers import SimpleRetriever
|
128
|
+
from .sources.declarative.schema import JsonFileSchemaLoader
|
129
|
+
from .sources.declarative.transformations.add_fields import AddedFieldDefinition, AddFields
|
130
|
+
from .sources.declarative.transformations.transformation import RecordTransformation
|
131
|
+
from .sources.declarative.types import FieldPointer
|
132
|
+
from .sources.declarative.yaml_declarative_source import YamlDeclarativeSource
|
133
|
+
from .sources.message import InMemoryMessageRepository, MessageRepository
|
134
|
+
from .sources.source import TState
|
135
|
+
from .sources.streams.availability_strategy import AvailabilityStrategy
|
136
|
+
from .sources.streams.call_rate import (
|
137
|
+
AbstractAPIBudget,
|
138
|
+
CachedLimiterSession,
|
139
|
+
HttpAPIBudget,
|
140
|
+
HttpRequestMatcher,
|
141
|
+
LimiterSession,
|
142
|
+
MovingWindowCallRatePolicy,
|
143
|
+
Rate,
|
144
|
+
)
|
145
|
+
from .sources.streams.checkpoint import Cursor as LegacyCursor
|
146
|
+
from .sources.streams.checkpoint import ResumableFullRefreshCursor
|
147
|
+
from .sources.streams.concurrent.adapters import StreamFacade
|
148
|
+
from .sources.streams.concurrent.cursor import (
|
149
|
+
ConcurrentCursor,
|
150
|
+
Cursor,
|
151
|
+
CursorField,
|
152
|
+
FinalStateCursor,
|
153
|
+
)
|
154
|
+
from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import (
|
155
|
+
EpochValueConcurrentStreamStateConverter,
|
156
|
+
IsoMillisConcurrentStreamStateConverter,
|
157
|
+
)
|
158
|
+
from .sources.streams.core import IncrementalMixin, Stream, package_name_from_class
|
159
|
+
from .sources.streams.http import HttpStream, HttpSubStream
|
160
|
+
from .sources.streams.http.availability_strategy import HttpAvailabilityStrategy
|
161
|
+
from .sources.streams.http.exceptions import (
|
162
|
+
BaseBackoffException,
|
163
|
+
DefaultBackoffException,
|
164
|
+
UserDefinedBackoffException,
|
165
|
+
)
|
166
|
+
from .sources.streams.http.rate_limiting import default_backoff_handler
|
167
|
+
from .sources.streams.http.requests_native_auth import (
|
168
|
+
Oauth2Authenticator,
|
169
|
+
SingleUseRefreshTokenOauth2Authenticator,
|
170
|
+
TokenAuthenticator,
|
171
|
+
)
|
172
|
+
from .sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
|
173
|
+
from .sources.types import Config, Record, StreamSlice
|
174
|
+
from .sources.utils import casing
|
175
|
+
from .sources.utils.schema_helpers import (
|
176
|
+
InternalConfig,
|
177
|
+
ResourceSchemaLoader,
|
178
|
+
check_config_against_spec_or_exit,
|
179
|
+
expand_refs,
|
180
|
+
split_config,
|
181
|
+
)
|
182
|
+
from .sources.utils.transform import TransformConfig, TypeTransformer
|
183
|
+
from .utils import AirbyteTracedException, is_cloud_environment
|
184
|
+
from .utils.constants import ENV_REQUEST_CACHE_PATH
|
185
|
+
from .utils.event_timing import create_timer
|
186
|
+
from .utils.oneof_option_config import OneOfOptionConfig
|
187
|
+
from .utils.spec_schema_transformations import resolve_refs
|
188
|
+
from .utils.stream_status_utils import as_airbyte_message
|
189
|
+
|
190
|
+
__all__ = [
|
191
|
+
# Availability strategy
|
192
|
+
"AvailabilityStrategy",
|
193
|
+
"HttpAvailabilityStrategy",
|
194
|
+
# Checkpoint
|
195
|
+
"LegacyCursor",
|
196
|
+
"ResumableFullRefreshCursor",
|
197
|
+
# Concurrent
|
198
|
+
"ConcurrentCursor",
|
199
|
+
"ConcurrentSource",
|
200
|
+
"ConcurrentSourceAdapter",
|
201
|
+
"Cursor",
|
202
|
+
"CursorField",
|
203
|
+
"DEFAULT_CONCURRENCY",
|
204
|
+
"EpochValueConcurrentStreamStateConverter",
|
205
|
+
"FinalStateCursor",
|
206
|
+
"IsoMillisConcurrentStreamStateConverter",
|
207
|
+
"StreamFacade",
|
208
|
+
# Config observation
|
209
|
+
"create_connector_config_control_message",
|
210
|
+
"emit_configuration_as_airbyte_control_message",
|
211
|
+
# Connector
|
212
|
+
"AbstractSource",
|
213
|
+
"BaseConfig",
|
214
|
+
"BaseConnector",
|
215
|
+
"Connector",
|
216
|
+
"Destination",
|
217
|
+
"Source",
|
218
|
+
"TState",
|
219
|
+
# Declarative
|
220
|
+
"AddFields",
|
221
|
+
"AddedFieldDefinition",
|
222
|
+
"ApiKeyAuthenticator",
|
223
|
+
"BackoffStrategy",
|
224
|
+
"BasicHttpAuthenticator",
|
225
|
+
"BearerAuthenticator",
|
226
|
+
"CartesianProductStreamSlicer",
|
227
|
+
"CursorPaginationStrategy",
|
228
|
+
"DatetimeBasedCursor",
|
229
|
+
"DeclarativeAuthenticator",
|
230
|
+
"DeclarativeOauth2Authenticator",
|
231
|
+
"DeclarativeSingleUseRefreshTokenOauth2Authenticator",
|
232
|
+
"DeclarativeStream",
|
233
|
+
"Decoder",
|
234
|
+
"DefaultPaginator",
|
235
|
+
"DefaultRequestOptionsProvider",
|
236
|
+
"DpathExtractor",
|
237
|
+
"FieldPointer",
|
238
|
+
"HttpMethod",
|
239
|
+
"HttpRequester",
|
240
|
+
"InterpolatedBoolean",
|
241
|
+
"InterpolatedRequestInputProvider",
|
242
|
+
"InterpolatedString",
|
243
|
+
"JsonDecoder",
|
244
|
+
"JsonFileSchemaLoader",
|
245
|
+
"LegacyToPerPartitionStateMigration",
|
246
|
+
"ManifestDeclarativeSource",
|
247
|
+
"MinMaxDatetime",
|
248
|
+
"NoAuth",
|
249
|
+
"OffsetIncrement",
|
250
|
+
"PageIncrement",
|
251
|
+
"PaginationStrategy",
|
252
|
+
"ParentStreamConfig",
|
253
|
+
"ReadException",
|
254
|
+
"RecordExtractor",
|
255
|
+
"RecordFilter",
|
256
|
+
"RecordSelector",
|
257
|
+
"RecordTransformation",
|
258
|
+
"RequestOption",
|
259
|
+
"RequestOptionType",
|
260
|
+
"Requester",
|
261
|
+
"ResponseStatus",
|
262
|
+
"SimpleRetriever",
|
263
|
+
"SinglePartitionRouter",
|
264
|
+
"StopConditionPaginationStrategyDecorator",
|
265
|
+
"StreamSlice",
|
266
|
+
"SubstreamPartitionRouter",
|
267
|
+
"YamlDeclarativeSource",
|
268
|
+
# Entrypoint
|
269
|
+
"launch",
|
270
|
+
"AirbyteEntrypoint",
|
271
|
+
# HTTP
|
272
|
+
"AbstractAPIBudget",
|
273
|
+
"AbstractHeaderAuthenticator",
|
274
|
+
"BaseBackoffException",
|
275
|
+
"CachedLimiterSession",
|
276
|
+
"DefaultBackoffException",
|
277
|
+
"default_backoff_handler",
|
278
|
+
"HttpAPIBudget",
|
279
|
+
"HttpAuthenticator",
|
280
|
+
"HttpRequestMatcher",
|
281
|
+
"HttpStream",
|
282
|
+
"HttpSubStream",
|
283
|
+
"LimiterSession",
|
284
|
+
"MovingWindowCallRatePolicy",
|
285
|
+
"MultipleTokenAuthenticator",
|
286
|
+
"Oauth2Authenticator",
|
287
|
+
"Rate",
|
288
|
+
"SingleUseRefreshTokenOauth2Authenticator",
|
289
|
+
"TokenAuthenticator",
|
290
|
+
"UserDefinedBackoffException",
|
291
|
+
# Logger
|
292
|
+
"AirbyteLogFormatter",
|
293
|
+
"init_logger",
|
294
|
+
# Protocol classes
|
295
|
+
"AirbyteStream",
|
296
|
+
"AirbyteConnectionStatus",
|
297
|
+
"AirbyteMessage",
|
298
|
+
"ConfiguredAirbyteCatalog",
|
299
|
+
"Status",
|
300
|
+
"Type",
|
301
|
+
"OrchestratorType",
|
302
|
+
"ConfiguredAirbyteStream",
|
303
|
+
"DestinationSyncMode",
|
304
|
+
"SyncMode",
|
305
|
+
"FailureType",
|
306
|
+
"AdvancedAuth",
|
307
|
+
"AirbyteLogMessage",
|
308
|
+
"OAuthConfigSpecification",
|
309
|
+
"ConnectorSpecification",
|
310
|
+
"Level",
|
311
|
+
"AirbyteRecordMessage",
|
312
|
+
# Repository
|
313
|
+
"InMemoryMessageRepository",
|
314
|
+
"MessageRepository",
|
315
|
+
# State management
|
316
|
+
"ConnectorStateManager",
|
317
|
+
# Stream
|
318
|
+
"IncrementalMixin",
|
319
|
+
"Stream",
|
320
|
+
"StreamData",
|
321
|
+
"package_name_from_class",
|
322
|
+
# Utils
|
323
|
+
"AirbyteTracedException",
|
324
|
+
"is_cloud_environment",
|
325
|
+
"casing",
|
326
|
+
"InternalConfig",
|
327
|
+
"ResourceSchemaLoader",
|
328
|
+
"check_config_against_spec_or_exit",
|
329
|
+
"split_config",
|
330
|
+
"TransformConfig",
|
331
|
+
"TypeTransformer",
|
332
|
+
"ENV_REQUEST_CACHE_PATH",
|
333
|
+
"create_timer",
|
334
|
+
"OneOfOptionConfig",
|
335
|
+
"resolve_refs",
|
336
|
+
"as_airbyte_message",
|
337
|
+
# Types
|
338
|
+
"Config",
|
339
|
+
"Record",
|
340
|
+
"Source",
|
341
|
+
"StreamSlice",
|
342
|
+
]
|
343
|
+
|
344
|
+
__version__: str
|
345
|
+
"""Version generated by poetry dynamic versioning during publish.
|
346
|
+
|
347
|
+
When running in development, dunamai will calculate a new prerelease version
|
348
|
+
from existing git release tag info.
|
349
|
+
"""
|
350
|
+
|
351
|
+
try:
|
352
|
+
__version__ = _dunamai.get_version(
|
353
|
+
"airbyte-cdk",
|
354
|
+
third_choice=_dunamai.Version.from_any_vcs,
|
355
|
+
fallback=_dunamai.Version("0.0.0+dev"),
|
356
|
+
).serialize()
|
357
|
+
except:
|
358
|
+
__version__ = "0.0.0+dev"
|
@@ -0,0 +1 @@
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
@@ -0,0 +1,230 @@
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
+
"""Defines the `source-declarative-manifest` connector, which installs alongside CDK.
|
3
|
+
|
4
|
+
This file was originally imported from the dedicated connector directory, under the
|
5
|
+
`airbyte` monorepo.
|
6
|
+
|
7
|
+
Usage:
|
8
|
+
|
9
|
+
```
|
10
|
+
pipx install airbyte-cdk
|
11
|
+
source-declarative-manifest --help
|
12
|
+
source-declarative-manifest spec
|
13
|
+
...
|
14
|
+
```
|
15
|
+
"""
|
16
|
+
|
17
|
+
from __future__ import annotations
|
18
|
+
|
19
|
+
import json
|
20
|
+
import pkgutil
|
21
|
+
import sys
|
22
|
+
import traceback
|
23
|
+
from collections.abc import Mapping
|
24
|
+
from datetime import datetime
|
25
|
+
from pathlib import Path
|
26
|
+
from typing import Any, cast
|
27
|
+
|
28
|
+
import orjson
|
29
|
+
|
30
|
+
from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
|
31
|
+
from airbyte_cdk.models import (
|
32
|
+
AirbyteErrorTraceMessage,
|
33
|
+
AirbyteMessage,
|
34
|
+
AirbyteMessageSerializer,
|
35
|
+
AirbyteStateMessage,
|
36
|
+
AirbyteTraceMessage,
|
37
|
+
ConfiguredAirbyteCatalog,
|
38
|
+
ConnectorSpecificationSerializer,
|
39
|
+
TraceType,
|
40
|
+
Type,
|
41
|
+
)
|
42
|
+
from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
|
43
|
+
ConcurrentDeclarativeSource,
|
44
|
+
)
|
45
|
+
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
|
46
|
+
from airbyte_cdk.sources.source import TState
|
47
|
+
|
48
|
+
|
49
|
+
class SourceLocalYaml(YamlDeclarativeSource):
|
50
|
+
"""
|
51
|
+
Declarative source defined by a yaml file in the local filesystem
|
52
|
+
"""
|
53
|
+
|
54
|
+
def __init__(
|
55
|
+
self,
|
56
|
+
catalog: ConfiguredAirbyteCatalog | None,
|
57
|
+
config: Mapping[str, Any] | None,
|
58
|
+
state: TState,
|
59
|
+
**kwargs: Any,
|
60
|
+
) -> None:
|
61
|
+
"""
|
62
|
+
HACK!
|
63
|
+
Problem: YamlDeclarativeSource relies on the calling module name/path to find the yaml file.
|
64
|
+
Implication: If you call YamlDeclarativeSource directly it will look for the yaml file in the wrong place. (e.g. the airbyte-cdk package)
|
65
|
+
Solution: Subclass YamlDeclarativeSource from the same location as the manifest to load.
|
66
|
+
|
67
|
+
When can we remove this?
|
68
|
+
When the airbyte-cdk is updated to not rely on the calling module name/path to find the yaml file.
|
69
|
+
When all manifest connectors are updated to use the new airbyte-cdk.
|
70
|
+
When all manifest connectors are updated to use the source-declarative-manifest as the base image.
|
71
|
+
"""
|
72
|
+
super().__init__(
|
73
|
+
catalog=catalog,
|
74
|
+
config=config,
|
75
|
+
state=state, # type: ignore [arg-type]
|
76
|
+
path_to_yaml="manifest.yaml",
|
77
|
+
)
|
78
|
+
|
79
|
+
|
80
|
+
def _is_local_manifest_command(args: list[str]) -> bool:
|
81
|
+
# Check for a local manifest.yaml file
|
82
|
+
return Path("/airbyte/integration_code/source_declarative_manifest/manifest.yaml").exists()
|
83
|
+
|
84
|
+
|
85
|
+
def handle_command(args: list[str]) -> None:
|
86
|
+
if _is_local_manifest_command(args):
|
87
|
+
handle_local_manifest_command(args)
|
88
|
+
else:
|
89
|
+
handle_remote_manifest_command(args)
|
90
|
+
|
91
|
+
|
92
|
+
def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
|
93
|
+
try:
|
94
|
+
config, catalog, state = _parse_inputs_into_config_catalog_state(args)
|
95
|
+
return SourceLocalYaml(config=config, catalog=catalog, state=state)
|
96
|
+
except Exception as error:
|
97
|
+
print(
|
98
|
+
orjson.dumps(
|
99
|
+
AirbyteMessageSerializer.dump(
|
100
|
+
AirbyteMessage(
|
101
|
+
type=Type.TRACE,
|
102
|
+
trace=AirbyteTraceMessage(
|
103
|
+
type=TraceType.ERROR,
|
104
|
+
emitted_at=int(datetime.now().timestamp() * 1000),
|
105
|
+
error=AirbyteErrorTraceMessage(
|
106
|
+
message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
|
107
|
+
stack_trace=traceback.format_exc(),
|
108
|
+
),
|
109
|
+
),
|
110
|
+
)
|
111
|
+
)
|
112
|
+
).decode()
|
113
|
+
)
|
114
|
+
raise error
|
115
|
+
|
116
|
+
|
117
|
+
def handle_local_manifest_command(args: list[str]) -> None:
|
118
|
+
source = _get_local_yaml_source(args)
|
119
|
+
launch(
|
120
|
+
source=source,
|
121
|
+
args=args,
|
122
|
+
)
|
123
|
+
|
124
|
+
|
125
|
+
def handle_remote_manifest_command(args: list[str]) -> None:
|
126
|
+
"""Overrides the spec command to return the generalized spec for the declarative manifest source.
|
127
|
+
|
128
|
+
This is different from a typical low-code, but built and published separately source built as a ManifestDeclarativeSource,
|
129
|
+
because that will have a spec method that returns the spec for that specific source. Other than spec,
|
130
|
+
the generalized connector behaves the same as any other, since the manifest is provided in the config.
|
131
|
+
"""
|
132
|
+
if args[0] == "spec":
|
133
|
+
json_spec = pkgutil.get_data(
|
134
|
+
"airbyte_cdk.cli.source_declarative_manifest",
|
135
|
+
"spec.json",
|
136
|
+
)
|
137
|
+
if json_spec is None:
|
138
|
+
raise FileNotFoundError(
|
139
|
+
"Could not find `spec.json` file for source-declarative-manifest"
|
140
|
+
)
|
141
|
+
|
142
|
+
spec_obj = json.loads(json_spec)
|
143
|
+
spec = ConnectorSpecificationSerializer.load(spec_obj)
|
144
|
+
|
145
|
+
message = AirbyteMessage(type=Type.SPEC, spec=spec)
|
146
|
+
print(AirbyteEntrypoint.airbyte_message_to_string(message))
|
147
|
+
else:
|
148
|
+
source = create_declarative_source(args)
|
149
|
+
launch(
|
150
|
+
source=source,
|
151
|
+
args=args,
|
152
|
+
)
|
153
|
+
|
154
|
+
|
155
|
+
def create_declarative_source(
|
156
|
+
args: list[str],
|
157
|
+
) -> ConcurrentDeclarativeSource: # type: ignore [type-arg]
|
158
|
+
"""Creates the source with the injected config.
|
159
|
+
|
160
|
+
This essentially does what other low-code sources do at build time, but at runtime,
|
161
|
+
with a user-provided manifest in the config. This better reflects what happens in the
|
162
|
+
connector builder.
|
163
|
+
"""
|
164
|
+
try:
|
165
|
+
config: Mapping[str, Any] | None
|
166
|
+
catalog: ConfiguredAirbyteCatalog | None
|
167
|
+
state: list[AirbyteStateMessage]
|
168
|
+
config, catalog, state = _parse_inputs_into_config_catalog_state(args)
|
169
|
+
if config is None or "__injected_declarative_manifest" not in config:
|
170
|
+
raise ValueError(
|
171
|
+
"Invalid config: `__injected_declarative_manifest` should be provided at the root "
|
172
|
+
f"of the config but config only has keys: {list(config.keys() if config else [])}"
|
173
|
+
)
|
174
|
+
return ConcurrentDeclarativeSource(
|
175
|
+
config=config,
|
176
|
+
catalog=catalog,
|
177
|
+
state=state,
|
178
|
+
source_config=cast(dict[str, Any], config["__injected_declarative_manifest"]),
|
179
|
+
)
|
180
|
+
except Exception as error:
|
181
|
+
print(
|
182
|
+
orjson.dumps(
|
183
|
+
AirbyteMessageSerializer.dump(
|
184
|
+
AirbyteMessage(
|
185
|
+
type=Type.TRACE,
|
186
|
+
trace=AirbyteTraceMessage(
|
187
|
+
type=TraceType.ERROR,
|
188
|
+
emitted_at=int(datetime.now().timestamp() * 1000),
|
189
|
+
error=AirbyteErrorTraceMessage(
|
190
|
+
message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
|
191
|
+
stack_trace=traceback.format_exc(),
|
192
|
+
),
|
193
|
+
),
|
194
|
+
)
|
195
|
+
)
|
196
|
+
).decode()
|
197
|
+
)
|
198
|
+
raise error
|
199
|
+
|
200
|
+
|
201
|
+
def _parse_inputs_into_config_catalog_state(
|
202
|
+
args: list[str],
|
203
|
+
) -> tuple[
|
204
|
+
Mapping[str, Any] | None,
|
205
|
+
ConfiguredAirbyteCatalog | None,
|
206
|
+
list[AirbyteStateMessage],
|
207
|
+
]:
|
208
|
+
parsed_args = AirbyteEntrypoint.parse_args(args)
|
209
|
+
config = (
|
210
|
+
ConcurrentDeclarativeSource.read_config(parsed_args.config)
|
211
|
+
if hasattr(parsed_args, "config")
|
212
|
+
else None
|
213
|
+
)
|
214
|
+
catalog = (
|
215
|
+
ConcurrentDeclarativeSource.read_catalog(parsed_args.catalog)
|
216
|
+
if hasattr(parsed_args, "catalog")
|
217
|
+
else None
|
218
|
+
)
|
219
|
+
state = (
|
220
|
+
ConcurrentDeclarativeSource.read_state(parsed_args.state)
|
221
|
+
if hasattr(parsed_args, "state")
|
222
|
+
else []
|
223
|
+
)
|
224
|
+
|
225
|
+
return config, catalog, state
|
226
|
+
|
227
|
+
|
228
|
+
def run() -> None:
|
229
|
+
args: list[str] = sys.argv[1:]
|
230
|
+
handle_command(args)
|
@@ -0,0 +1,17 @@
|
|
1
|
+
{
|
2
|
+
"documentationUrl": "https://docs.airbyte.com/integrations/sources/low-code",
|
3
|
+
"connectionSpecification": {
|
4
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
5
|
+
"title": "Low-code source spec",
|
6
|
+
"type": "object",
|
7
|
+
"required": ["__injected_declarative_manifest"],
|
8
|
+
"additionalProperties": true,
|
9
|
+
"properties": {
|
10
|
+
"__injected_declarative_manifest": {
|
11
|
+
"title": "Low-code manifest",
|
12
|
+
"type": "object",
|
13
|
+
"description": "The low-code manifest that defines the components of the source."
|
14
|
+
}
|
15
|
+
}
|
16
|
+
}
|
17
|
+
}
|
@@ -7,14 +7,29 @@ from __future__ import ( # Used to evaluate type hints at runtime, a NameError:
|
|
7
7
|
)
|
8
8
|
|
9
9
|
import time
|
10
|
+
from copy import copy
|
10
11
|
from typing import Any, List, MutableMapping
|
11
12
|
|
12
|
-
|
13
|
+
import orjson
|
14
|
+
|
15
|
+
from airbyte_cdk.models import (
|
16
|
+
AirbyteControlConnectorConfigMessage,
|
17
|
+
AirbyteControlMessage,
|
18
|
+
AirbyteMessage,
|
19
|
+
AirbyteMessageSerializer,
|
20
|
+
OrchestratorType,
|
21
|
+
Type,
|
22
|
+
)
|
13
23
|
|
14
24
|
|
15
|
-
class ObservedDict(dict):
|
16
|
-
def __init__(
|
17
|
-
|
25
|
+
class ObservedDict(dict): # type: ignore # disallow_any_generics is set to True, and dict is equivalent to dict[Any]
|
26
|
+
def __init__(
|
27
|
+
self,
|
28
|
+
non_observed_mapping: MutableMapping[Any, Any],
|
29
|
+
observer: ConfigObserver,
|
30
|
+
update_on_unchanged_value: bool = True,
|
31
|
+
) -> None:
|
32
|
+
non_observed_mapping = copy(non_observed_mapping)
|
18
33
|
self.observer = observer
|
19
34
|
self.update_on_unchanged_value = update_on_unchanged_value
|
20
35
|
for item, value in non_observed_mapping.items():
|
@@ -29,7 +44,7 @@ class ObservedDict(dict):
|
|
29
44
|
value[i] = ObservedDict(sub_value, observer)
|
30
45
|
super().__init__(non_observed_mapping)
|
31
46
|
|
32
|
-
def __setitem__(self, item: Any, value: Any):
|
47
|
+
def __setitem__(self, item: Any, value: Any) -> None:
|
33
48
|
"""Override dict.__setitem__ by:
|
34
49
|
1. Observing the new value if it is a dict
|
35
50
|
2. Call observer update if the new value is different from the previous one
|
@@ -58,25 +73,29 @@ class ConfigObserver:
|
|
58
73
|
emit_configuration_as_airbyte_control_message(self.config)
|
59
74
|
|
60
75
|
|
61
|
-
def observe_connector_config(
|
76
|
+
def observe_connector_config(
|
77
|
+
non_observed_connector_config: MutableMapping[str, Any],
|
78
|
+
) -> ObservedDict:
|
62
79
|
if isinstance(non_observed_connector_config, ObservedDict):
|
63
80
|
raise ValueError("This connector configuration is already observed")
|
64
81
|
connector_config_observer = ConfigObserver()
|
65
|
-
observed_connector_config = ObservedDict(
|
82
|
+
observed_connector_config = ObservedDict(
|
83
|
+
non_observed_connector_config, connector_config_observer
|
84
|
+
)
|
66
85
|
connector_config_observer.set_config(observed_connector_config)
|
67
86
|
return observed_connector_config
|
68
87
|
|
69
88
|
|
70
|
-
def emit_configuration_as_airbyte_control_message(config: MutableMapping):
|
89
|
+
def emit_configuration_as_airbyte_control_message(config: MutableMapping[str, Any]) -> None:
|
71
90
|
"""
|
72
91
|
WARNING: deprecated - emit_configuration_as_airbyte_control_message is being deprecated in favor of the MessageRepository mechanism.
|
73
92
|
See the airbyte_cdk.sources.message package
|
74
93
|
"""
|
75
94
|
airbyte_message = create_connector_config_control_message(config)
|
76
|
-
print(airbyte_message.
|
95
|
+
print(orjson.dumps(AirbyteMessageSerializer.dump(airbyte_message)).decode())
|
77
96
|
|
78
97
|
|
79
|
-
def create_connector_config_control_message(config):
|
98
|
+
def create_connector_config_control_message(config: MutableMapping[str, Any]) -> AirbyteMessage:
|
80
99
|
control_message = AirbyteControlMessage(
|
81
100
|
type=OrchestratorType.CONNECTOR_CONFIG,
|
82
101
|
emitted_at=time.time() * 1000,
|