airbyte-cdk 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/__init__.py +358 -0
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +236 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +104 -0
- airbyte_cdk/connector.py +123 -0
- airbyte_cdk/connector_builder/README.md +53 -0
- airbyte_cdk/connector_builder/__init__.py +3 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +121 -0
- airbyte_cdk/connector_builder/main.py +107 -0
- airbyte_cdk/connector_builder/models.py +73 -0
- airbyte_cdk/connector_builder/test_reader/__init__.py +7 -0
- airbyte_cdk/connector_builder/test_reader/helpers.py +689 -0
- airbyte_cdk/connector_builder/test_reader/message_grouper.py +173 -0
- airbyte_cdk/connector_builder/test_reader/reader.py +441 -0
- airbyte_cdk/connector_builder/test_reader/types.py +83 -0
- airbyte_cdk/destinations/__init__.py +8 -0
- airbyte_cdk/destinations/destination.py +154 -0
- airbyte_cdk/destinations/vector_db_based/README.md +37 -0
- airbyte_cdk/destinations/vector_db_based/__init__.py +38 -0
- airbyte_cdk/destinations/vector_db_based/config.py +298 -0
- airbyte_cdk/destinations/vector_db_based/document_processor.py +223 -0
- airbyte_cdk/destinations/vector_db_based/embedder.py +303 -0
- airbyte_cdk/destinations/vector_db_based/indexer.py +78 -0
- airbyte_cdk/destinations/vector_db_based/test_utils.py +63 -0
- airbyte_cdk/destinations/vector_db_based/utils.py +35 -0
- airbyte_cdk/destinations/vector_db_based/writer.py +104 -0
- airbyte_cdk/entrypoint.py +414 -0
- airbyte_cdk/exception_handler.py +56 -0
- airbyte_cdk/logger.py +109 -0
- airbyte_cdk/models/__init__.py +72 -0
- airbyte_cdk/models/airbyte_protocol.py +88 -0
- airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
- airbyte_cdk/models/well_known_types.py +5 -0
- airbyte_cdk/py.typed +0 -0
- airbyte_cdk/sources/__init__.py +26 -0
- airbyte_cdk/sources/abstract_source.py +326 -0
- airbyte_cdk/sources/concurrent_source/__init__.py +8 -0
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +255 -0
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +165 -0
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +147 -0
- airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py +24 -0
- airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +115 -0
- airbyte_cdk/sources/config.py +27 -0
- airbyte_cdk/sources/connector_state_manager.py +161 -0
- airbyte_cdk/sources/declarative/__init__.py +3 -0
- airbyte_cdk/sources/declarative/async_job/__init__.py +0 -0
- airbyte_cdk/sources/declarative/async_job/job.py +52 -0
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +525 -0
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +79 -0
- airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
- airbyte_cdk/sources/declarative/async_job/status.py +24 -0
- airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
- airbyte_cdk/sources/declarative/auth/__init__.py +8 -0
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +42 -0
- airbyte_cdk/sources/declarative/auth/jwt.py +197 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +293 -0
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +45 -0
- airbyte_cdk/sources/declarative/auth/token.py +267 -0
- airbyte_cdk/sources/declarative/auth/token_provider.py +82 -0
- airbyte_cdk/sources/declarative/checks/__init__.py +24 -0
- airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +61 -0
- airbyte_cdk/sources/declarative/checks/check_stream.py +56 -0
- airbyte_cdk/sources/declarative/checks/connection_checker.py +35 -0
- airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +526 -0
- airbyte_cdk/sources/declarative/datetime/__init__.py +3 -0
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +65 -0
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +118 -0
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +3975 -0
- airbyte_cdk/sources/declarative/declarative_source.py +36 -0
- airbyte_cdk/sources/declarative/declarative_stream.py +241 -0
- airbyte_cdk/sources/declarative/decoders/__init__.py +33 -0
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +218 -0
- airbyte_cdk/sources/declarative/decoders/decoder.py +32 -0
- airbyte_cdk/sources/declarative/decoders/decoder_parser.py +30 -0
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +65 -0
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
- airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py +56 -0
- airbyte_cdk/sources/declarative/exceptions.py +9 -0
- airbyte_cdk/sources/declarative/extractors/__init__.py +21 -0
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +86 -0
- airbyte_cdk/sources/declarative/extractors/http_selector.py +37 -0
- airbyte_cdk/sources/declarative/extractors/record_extractor.py +27 -0
- airbyte_cdk/sources/declarative/extractors/record_filter.py +91 -0
- airbyte_cdk/sources/declarative/extractors/record_selector.py +170 -0
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +176 -0
- airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
- airbyte_cdk/sources/declarative/incremental/__init__.py +37 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +497 -0
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +459 -0
- airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +357 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +380 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
- airbyte_cdk/sources/declarative/interpolation/__init__.py +9 -0
- airbyte_cdk/sources/declarative/interpolation/filters.py +139 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +66 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +56 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +52 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +79 -0
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +34 -0
- airbyte_cdk/sources/declarative/interpolation/jinja.py +161 -0
- airbyte_cdk/sources/declarative/interpolation/macros.py +191 -0
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +421 -0
- airbyte_cdk/sources/declarative/migrations/__init__.py +0 -0
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
- airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
- airbyte_cdk/sources/declarative/models/__init__.py +2 -0
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2503 -0
- airbyte_cdk/sources/declarative/parsers/__init__.py +3 -0
- airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +157 -0
- airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +21 -0
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +172 -0
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +213 -0
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +3407 -0
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +29 -0
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +121 -0
- airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
- airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +63 -0
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +437 -0
- airbyte_cdk/sources/declarative/requesters/README.md +56 -0
- airbyte_cdk/sources/declarative/requesters/__init__.py +9 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +25 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +23 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +45 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +45 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +41 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +70 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +77 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +17 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +101 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +147 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +17 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +179 -0
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +350 -0
- airbyte_cdk/sources/declarative/requesters/http_requester.py +433 -0
- airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +21 -0
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +327 -0
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +76 -0
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +65 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +25 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +98 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +102 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +71 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +48 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +66 -0
- airbyte_cdk/sources/declarative/requesters/request_option.py +117 -0
- airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +23 -0
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +92 -0
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +59 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +68 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +119 -0
- airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +79 -0
- airbyte_cdk/sources/declarative/requesters/request_path.py +15 -0
- airbyte_cdk/sources/declarative/requesters/requester.py +144 -0
- airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
- airbyte_cdk/sources/declarative/retrievers/__init__.py +19 -0
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +124 -0
- airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
- airbyte_cdk/sources/declarative/retrievers/retriever.py +54 -0
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +702 -0
- airbyte_cdk/sources/declarative/schema/__init__.py +25 -0
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +47 -0
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +285 -0
- airbyte_cdk/sources/declarative/schema/inline_schema_loader.py +19 -0
- airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +92 -0
- airbyte_cdk/sources/declarative/schema/schema_loader.py +17 -0
- airbyte_cdk/sources/declarative/spec/__init__.py +7 -0
- airbyte_cdk/sources/declarative/spec/spec.py +48 -0
- airbyte_cdk/sources/declarative/stream_slicers/__init__.py +7 -0
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +93 -0
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +25 -0
- airbyte_cdk/sources/declarative/transformations/__init__.py +17 -0
- airbyte_cdk/sources/declarative/transformations/add_fields.py +146 -0
- airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py +61 -0
- airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
- airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +75 -0
- airbyte_cdk/sources/declarative/transformations/transformation.py +37 -0
- airbyte_cdk/sources/declarative/types.py +25 -0
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +67 -0
- airbyte_cdk/sources/file_based/README.md +152 -0
- airbyte_cdk/sources/file_based/__init__.py +24 -0
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +11 -0
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +73 -0
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +149 -0
- airbyte_cdk/sources/file_based/config/__init__.py +0 -0
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +153 -0
- airbyte_cdk/sources/file_based/config/avro_format.py +25 -0
- airbyte_cdk/sources/file_based/config/csv_format.py +210 -0
- airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +99 -0
- airbyte_cdk/sources/file_based/config/jsonl_format.py +18 -0
- airbyte_cdk/sources/file_based/config/parquet_format.py +25 -0
- airbyte_cdk/sources/file_based/config/unstructured_format.py +102 -0
- airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py +81 -0
- airbyte_cdk/sources/file_based/discovery_policy/__init__.py +8 -0
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +21 -0
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +33 -0
- airbyte_cdk/sources/file_based/exceptions.py +159 -0
- airbyte_cdk/sources/file_based/file_based_source.py +466 -0
- airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py +123 -0
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +209 -0
- airbyte_cdk/sources/file_based/file_record_data.py +22 -0
- airbyte_cdk/sources/file_based/file_types/__init__.py +37 -0
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +233 -0
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +527 -0
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +30 -0
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +86 -0
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +145 -0
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +275 -0
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +480 -0
- airbyte_cdk/sources/file_based/remote_file.py +18 -0
- airbyte_cdk/sources/file_based/schema_helpers.py +281 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +17 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +20 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +52 -0
- airbyte_cdk/sources/file_based/stream/__init__.py +13 -0
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +197 -0
- airbyte_cdk/sources/file_based/stream/concurrent/__init__.py +0 -0
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +343 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +9 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +59 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +313 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +83 -0
- airbyte_cdk/sources/file_based/stream/cursor/__init__.py +4 -0
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +66 -0
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +149 -0
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +396 -0
- airbyte_cdk/sources/file_based/stream/identities_stream.py +49 -0
- airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +92 -0
- airbyte_cdk/sources/file_based/types.py +10 -0
- airbyte_cdk/sources/http_config.py +10 -0
- airbyte_cdk/sources/http_logger.py +55 -0
- airbyte_cdk/sources/message/__init__.py +19 -0
- airbyte_cdk/sources/message/repository.py +137 -0
- airbyte_cdk/sources/source.py +95 -0
- airbyte_cdk/sources/specs/transfer_modes.py +26 -0
- airbyte_cdk/sources/streams/__init__.py +8 -0
- airbyte_cdk/sources/streams/availability_strategy.py +84 -0
- airbyte_cdk/sources/streams/call_rate.py +704 -0
- airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
- airbyte_cdk/sources/streams/checkpoint/cursor.py +77 -0
- airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
- airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
- airbyte_cdk/sources/streams/concurrent/README.md +7 -0
- airbyte_cdk/sources/streams/concurrent/__init__.py +3 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +96 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py +37 -0
- airbyte_cdk/sources/streams/concurrent/adapters.py +397 -0
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +94 -0
- airbyte_cdk/sources/streams/concurrent/clamping.py +99 -0
- airbyte_cdk/sources/streams/concurrent/cursor.py +481 -0
- airbyte_cdk/sources/streams/concurrent/cursor_types.py +32 -0
- airbyte_cdk/sources/streams/concurrent/default_stream.py +102 -0
- airbyte_cdk/sources/streams/concurrent/exceptions.py +18 -0
- airbyte_cdk/sources/streams/concurrent/helpers.py +42 -0
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +64 -0
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +45 -0
- airbyte_cdk/sources/streams/concurrent/partitions/__init__.py +3 -0
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +48 -0
- airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py +18 -0
- airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +38 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py +0 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +182 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +223 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
- airbyte_cdk/sources/streams/core.py +703 -0
- airbyte_cdk/sources/streams/http/__init__.py +10 -0
- airbyte_cdk/sources/streams/http/availability_strategy.py +54 -0
- airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
- airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
- airbyte_cdk/sources/streams/http/exceptions.py +61 -0
- airbyte_cdk/sources/streams/http/http.py +673 -0
- airbyte_cdk/sources/streams/http/http_client.py +531 -0
- airbyte_cdk/sources/streams/http/rate_limiting.py +158 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py +14 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +479 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +34 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +436 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +83 -0
- airbyte_cdk/sources/streams/permissions/identities_stream.py +75 -0
- airbyte_cdk/sources/streams/utils/__init__.py +3 -0
- airbyte_cdk/sources/types.py +169 -0
- airbyte_cdk/sources/utils/__init__.py +7 -0
- airbyte_cdk/sources/utils/casing.py +12 -0
- airbyte_cdk/sources/utils/files_directory.py +15 -0
- airbyte_cdk/sources/utils/record_helper.py +53 -0
- airbyte_cdk/sources/utils/schema_helpers.py +230 -0
- airbyte_cdk/sources/utils/slice_logger.py +57 -0
- airbyte_cdk/sources/utils/transform.py +277 -0
- airbyte_cdk/sources/utils/types.py +7 -0
- airbyte_cdk/sql/__init__.py +0 -0
- airbyte_cdk/sql/_util/__init__.py +0 -0
- airbyte_cdk/sql/_util/hashing.py +34 -0
- airbyte_cdk/sql/_util/name_normalizers.py +92 -0
- airbyte_cdk/sql/constants.py +32 -0
- airbyte_cdk/sql/exceptions.py +235 -0
- airbyte_cdk/sql/secrets.py +123 -0
- airbyte_cdk/sql/shared/__init__.py +15 -0
- airbyte_cdk/sql/shared/catalog_providers.py +145 -0
- airbyte_cdk/sql/shared/sql_processor.py +786 -0
- airbyte_cdk/sql/types.py +160 -0
- airbyte_cdk/test/__init__.py +7 -0
- airbyte_cdk/test/catalog_builder.py +81 -0
- airbyte_cdk/test/entrypoint_wrapper.py +250 -0
- airbyte_cdk/test/mock_http/__init__.py +6 -0
- airbyte_cdk/test/mock_http/matcher.py +41 -0
- airbyte_cdk/test/mock_http/mocker.py +185 -0
- airbyte_cdk/test/mock_http/request.py +103 -0
- airbyte_cdk/test/mock_http/response.py +28 -0
- airbyte_cdk/test/mock_http/response_builder.py +237 -0
- airbyte_cdk/test/state_builder.py +33 -0
- airbyte_cdk/test/utils/__init__.py +1 -0
- airbyte_cdk/test/utils/data.py +24 -0
- airbyte_cdk/test/utils/http_mocking.py +16 -0
- airbyte_cdk/test/utils/manifest_only_fixtures.py +59 -0
- airbyte_cdk/test/utils/reading.py +26 -0
- airbyte_cdk/utils/__init__.py +10 -0
- airbyte_cdk/utils/airbyte_secrets_utils.py +80 -0
- airbyte_cdk/utils/analytics_message.py +25 -0
- airbyte_cdk/utils/constants.py +5 -0
- airbyte_cdk/utils/datetime_format_inferrer.py +94 -0
- airbyte_cdk/utils/datetime_helpers.py +499 -0
- airbyte_cdk/utils/event_timing.py +85 -0
- airbyte_cdk/utils/is_cloud_environment.py +18 -0
- airbyte_cdk/utils/mapping_helpers.py +162 -0
- airbyte_cdk/utils/message_utils.py +26 -0
- airbyte_cdk/utils/oneof_option_config.py +33 -0
- airbyte_cdk/utils/print_buffer.py +75 -0
- airbyte_cdk/utils/schema_inferrer.py +270 -0
- airbyte_cdk/utils/slice_hasher.py +37 -0
- airbyte_cdk/utils/spec_schema_transformations.py +26 -0
- airbyte_cdk/utils/stream_status_utils.py +43 -0
- airbyte_cdk/utils/traced_exception.py +145 -0
- airbyte_cdk-0.0.0.dev0.dist-info/LICENSE.txt +19 -0
- airbyte_cdk-0.0.0.dev0.dist-info/LICENSE_SHORT +1 -0
- airbyte_cdk-0.0.0.dev0.dist-info/METADATA +111 -0
- airbyte_cdk-0.0.0.dev0.dist-info/RECORD +368 -0
- airbyte_cdk-0.0.0.dev0.dist-info/WHEEL +4 -0
- airbyte_cdk-0.0.0.dev0.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
import copy
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
from dataclasses import InitVar, dataclass
|
|
10
|
+
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional, Union
|
|
11
|
+
|
|
12
|
+
import dpath
|
|
13
|
+
import requests
|
|
14
|
+
|
|
15
|
+
from airbyte_cdk.models import AirbyteMessage
|
|
16
|
+
from airbyte_cdk.models import Type as MessageType
|
|
17
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
|
18
|
+
from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter
|
|
19
|
+
from airbyte_cdk.sources.declarative.requesters.request_option import (
|
|
20
|
+
RequestOption,
|
|
21
|
+
RequestOptionType,
|
|
22
|
+
)
|
|
23
|
+
from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
|
|
24
|
+
from airbyte_cdk.utils import AirbyteTracedException
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class ParentStreamConfig:
|
|
32
|
+
"""
|
|
33
|
+
Describes how to create a stream slice from a parent stream
|
|
34
|
+
|
|
35
|
+
stream: The stream to read records from
|
|
36
|
+
parent_key: The key of the parent stream's records that will be the stream slice key
|
|
37
|
+
partition_field: The partition key
|
|
38
|
+
extra_fields: Additional field paths to include in the stream slice
|
|
39
|
+
request_option: How to inject the slice value on an outgoing HTTP request
|
|
40
|
+
incremental_dependency (bool): Indicates if the parent stream should be read incrementally.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
stream: "DeclarativeStream" # Parent streams must be DeclarativeStream because we can't know which part of the stream slice is a partition for regular Stream
|
|
44
|
+
parent_key: Union[InterpolatedString, str]
|
|
45
|
+
partition_field: Union[InterpolatedString, str]
|
|
46
|
+
config: Config
|
|
47
|
+
parameters: InitVar[Mapping[str, Any]]
|
|
48
|
+
extra_fields: Optional[Union[List[List[str]], List[List[InterpolatedString]]]] = (
|
|
49
|
+
None # List of field paths (arrays of strings)
|
|
50
|
+
)
|
|
51
|
+
request_option: Optional[RequestOption] = None
|
|
52
|
+
incremental_dependency: bool = False
|
|
53
|
+
lazy_read_pointer: Optional[List[Union[InterpolatedString, str]]] = None
|
|
54
|
+
|
|
55
|
+
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
|
56
|
+
self.parent_key = InterpolatedString.create(self.parent_key, parameters=parameters)
|
|
57
|
+
self.partition_field = InterpolatedString.create(
|
|
58
|
+
self.partition_field, parameters=parameters
|
|
59
|
+
)
|
|
60
|
+
if self.extra_fields:
|
|
61
|
+
# Create InterpolatedString for each field path in extra_keys
|
|
62
|
+
self.extra_fields = [
|
|
63
|
+
[InterpolatedString.create(path, parameters=parameters) for path in key_path]
|
|
64
|
+
for key_path in self.extra_fields
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
self.lazy_read_pointer = (
|
|
68
|
+
[
|
|
69
|
+
InterpolatedString.create(path, parameters=parameters)
|
|
70
|
+
if isinstance(path, str)
|
|
71
|
+
else path
|
|
72
|
+
for path in self.lazy_read_pointer
|
|
73
|
+
]
|
|
74
|
+
if self.lazy_read_pointer
|
|
75
|
+
else None
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class SubstreamPartitionRouter(PartitionRouter):
|
|
81
|
+
"""
|
|
82
|
+
Partition router that iterates over the parent's stream records and emits slices
|
|
83
|
+
Will populate the state with `partition_field` and `parent_slice` so they can be accessed by other components
|
|
84
|
+
|
|
85
|
+
Attributes:
|
|
86
|
+
parent_stream_configs (List[ParentStreamConfig]): parent streams to iterate over and their config
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
parent_stream_configs: List[ParentStreamConfig]
|
|
90
|
+
config: Config
|
|
91
|
+
parameters: InitVar[Mapping[str, Any]]
|
|
92
|
+
|
|
93
|
+
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
|
94
|
+
if not self.parent_stream_configs:
|
|
95
|
+
raise ValueError("SubstreamPartitionRouter needs at least 1 parent stream")
|
|
96
|
+
self._parameters = parameters
|
|
97
|
+
|
|
98
|
+
def get_request_params(
|
|
99
|
+
self,
|
|
100
|
+
stream_state: Optional[StreamState] = None,
|
|
101
|
+
stream_slice: Optional[StreamSlice] = None,
|
|
102
|
+
next_page_token: Optional[Mapping[str, Any]] = None,
|
|
103
|
+
) -> Mapping[str, Any]:
|
|
104
|
+
# Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response
|
|
105
|
+
return self._get_request_option(RequestOptionType.request_parameter, stream_slice)
|
|
106
|
+
|
|
107
|
+
def get_request_headers(
|
|
108
|
+
self,
|
|
109
|
+
stream_state: Optional[StreamState] = None,
|
|
110
|
+
stream_slice: Optional[StreamSlice] = None,
|
|
111
|
+
next_page_token: Optional[Mapping[str, Any]] = None,
|
|
112
|
+
) -> Mapping[str, Any]:
|
|
113
|
+
# Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response
|
|
114
|
+
return self._get_request_option(RequestOptionType.header, stream_slice)
|
|
115
|
+
|
|
116
|
+
def get_request_body_data(
|
|
117
|
+
self,
|
|
118
|
+
stream_state: Optional[StreamState] = None,
|
|
119
|
+
stream_slice: Optional[StreamSlice] = None,
|
|
120
|
+
next_page_token: Optional[Mapping[str, Any]] = None,
|
|
121
|
+
) -> Mapping[str, Any]:
|
|
122
|
+
# Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response
|
|
123
|
+
return self._get_request_option(RequestOptionType.body_data, stream_slice)
|
|
124
|
+
|
|
125
|
+
def get_request_body_json(
|
|
126
|
+
self,
|
|
127
|
+
stream_state: Optional[StreamState] = None,
|
|
128
|
+
stream_slice: Optional[StreamSlice] = None,
|
|
129
|
+
next_page_token: Optional[Mapping[str, Any]] = None,
|
|
130
|
+
) -> Mapping[str, Any]:
|
|
131
|
+
# Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response
|
|
132
|
+
return self._get_request_option(RequestOptionType.body_json, stream_slice)
|
|
133
|
+
|
|
134
|
+
def _get_request_option(
|
|
135
|
+
self, option_type: RequestOptionType, stream_slice: Optional[StreamSlice]
|
|
136
|
+
) -> Mapping[str, Any]:
|
|
137
|
+
params: MutableMapping[str, Any] = {}
|
|
138
|
+
if stream_slice:
|
|
139
|
+
for parent_config in self.parent_stream_configs:
|
|
140
|
+
if (
|
|
141
|
+
parent_config.request_option
|
|
142
|
+
and parent_config.request_option.inject_into == option_type
|
|
143
|
+
):
|
|
144
|
+
key = parent_config.partition_field.eval(self.config) # type: ignore # partition_field is always casted to an interpolated string
|
|
145
|
+
value = stream_slice.get(key)
|
|
146
|
+
if value:
|
|
147
|
+
parent_config.request_option.inject_into_request(params, value, self.config)
|
|
148
|
+
return params
|
|
149
|
+
|
|
150
|
+
def stream_slices(self) -> Iterable[StreamSlice]:
|
|
151
|
+
"""
|
|
152
|
+
Iterate over each parent stream's record and create a StreamSlice for each record.
|
|
153
|
+
|
|
154
|
+
For each stream, iterate over its stream_slices.
|
|
155
|
+
For each stream slice, iterate over each record.
|
|
156
|
+
yield a stream slice for each such records.
|
|
157
|
+
|
|
158
|
+
If a parent slice contains no record, emit a slice with parent_record=None.
|
|
159
|
+
|
|
160
|
+
The template string can interpolate the following values:
|
|
161
|
+
- parent_stream_slice: mapping representing the parent's stream slice
|
|
162
|
+
- parent_record: mapping representing the parent record
|
|
163
|
+
- parent_stream_name: string representing the parent stream name
|
|
164
|
+
"""
|
|
165
|
+
if not self.parent_stream_configs:
|
|
166
|
+
yield from []
|
|
167
|
+
else:
|
|
168
|
+
for parent_stream_config in self.parent_stream_configs:
|
|
169
|
+
parent_stream = parent_stream_config.stream
|
|
170
|
+
parent_field = parent_stream_config.parent_key.eval(self.config) # type: ignore # parent_key is always casted to an interpolated string
|
|
171
|
+
partition_field = parent_stream_config.partition_field.eval(self.config) # type: ignore # partition_field is always casted to an interpolated string
|
|
172
|
+
extra_fields = None
|
|
173
|
+
if parent_stream_config.extra_fields:
|
|
174
|
+
extra_fields = [
|
|
175
|
+
[field_path_part.eval(self.config) for field_path_part in field_path] # type: ignore [union-attr]
|
|
176
|
+
for field_path in parent_stream_config.extra_fields
|
|
177
|
+
]
|
|
178
|
+
|
|
179
|
+
# read_stateless() assumes the parent is not concurrent. This is currently okay since the concurrent CDK does
|
|
180
|
+
# not support either substreams or RFR, but something that needs to be considered once we do
|
|
181
|
+
for parent_record in parent_stream.read_only_records():
|
|
182
|
+
parent_partition = None
|
|
183
|
+
# Skip non-records (eg AirbyteLogMessage)
|
|
184
|
+
if isinstance(parent_record, AirbyteMessage):
|
|
185
|
+
self.logger.warning(
|
|
186
|
+
f"Parent stream {parent_stream.name} returns records of type AirbyteMessage. This SubstreamPartitionRouter is not able to checkpoint incremental parent state."
|
|
187
|
+
)
|
|
188
|
+
if parent_record.type == MessageType.RECORD:
|
|
189
|
+
parent_record = parent_record.record.data # type: ignore[union-attr, assignment] # record is always a Record
|
|
190
|
+
else:
|
|
191
|
+
continue
|
|
192
|
+
elif isinstance(parent_record, Record):
|
|
193
|
+
parent_partition = (
|
|
194
|
+
parent_record.associated_slice.partition
|
|
195
|
+
if parent_record.associated_slice
|
|
196
|
+
else {}
|
|
197
|
+
)
|
|
198
|
+
parent_record = parent_record.data
|
|
199
|
+
elif not isinstance(parent_record, Mapping):
|
|
200
|
+
# The parent_record should only take the form of a Record, AirbyteMessage, or Mapping. Anything else is invalid
|
|
201
|
+
raise AirbyteTracedException(
|
|
202
|
+
message=f"Parent stream returned records as invalid type {type(parent_record)}"
|
|
203
|
+
)
|
|
204
|
+
try:
|
|
205
|
+
partition_value = dpath.get(
|
|
206
|
+
parent_record, # type: ignore [arg-type]
|
|
207
|
+
parent_field,
|
|
208
|
+
)
|
|
209
|
+
except KeyError:
|
|
210
|
+
continue
|
|
211
|
+
|
|
212
|
+
# Add extra fields
|
|
213
|
+
extracted_extra_fields = self._extract_extra_fields(parent_record, extra_fields)
|
|
214
|
+
|
|
215
|
+
if parent_stream_config.lazy_read_pointer:
|
|
216
|
+
extracted_extra_fields = {
|
|
217
|
+
"child_response": self._extract_child_response(
|
|
218
|
+
parent_record,
|
|
219
|
+
parent_stream_config.lazy_read_pointer, # type: ignore[arg-type] # lazy_read_pointer type handeled in __post_init__ of parent_stream_config
|
|
220
|
+
),
|
|
221
|
+
**extracted_extra_fields,
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
yield StreamSlice(
|
|
225
|
+
partition={
|
|
226
|
+
partition_field: partition_value,
|
|
227
|
+
"parent_slice": parent_partition or {},
|
|
228
|
+
},
|
|
229
|
+
cursor_slice={},
|
|
230
|
+
extra_fields=extracted_extra_fields,
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
def _extract_child_response(
|
|
234
|
+
self, parent_record: Mapping[str, Any] | AirbyteMessage, pointer: List[InterpolatedString]
|
|
235
|
+
) -> requests.Response:
|
|
236
|
+
"""Extract child records from a parent record based on lazy pointers."""
|
|
237
|
+
|
|
238
|
+
def _create_response(data: MutableMapping[str, Any]) -> SafeResponse:
|
|
239
|
+
"""Create a SafeResponse with the given data."""
|
|
240
|
+
response = SafeResponse()
|
|
241
|
+
response.content = json.dumps(data).encode("utf-8")
|
|
242
|
+
response.status_code = 200
|
|
243
|
+
return response
|
|
244
|
+
|
|
245
|
+
path = [path.eval(self.config) for path in pointer]
|
|
246
|
+
return _create_response(dpath.get(parent_record, path, default=[])) # type: ignore # argunet will be a MutableMapping, given input data structure
|
|
247
|
+
|
|
248
|
+
def _extract_extra_fields(
|
|
249
|
+
self,
|
|
250
|
+
parent_record: Mapping[str, Any] | AirbyteMessage,
|
|
251
|
+
extra_fields: Optional[List[List[str]]] = None,
|
|
252
|
+
) -> Mapping[str, Any]:
|
|
253
|
+
"""
|
|
254
|
+
Extracts additional fields specified by their paths from the parent record.
|
|
255
|
+
|
|
256
|
+
Args:
|
|
257
|
+
parent_record (Mapping[str, Any]): The record from the parent stream to extract fields from.
|
|
258
|
+
extra_fields (Optional[List[List[str]]]): A list of field paths (as lists of strings) to extract from the parent record.
|
|
259
|
+
|
|
260
|
+
Returns:
|
|
261
|
+
Mapping[str, Any]: A dictionary containing the extracted fields.
|
|
262
|
+
The keys are the joined field paths, and the values are the corresponding extracted values.
|
|
263
|
+
"""
|
|
264
|
+
extracted_extra_fields = {}
|
|
265
|
+
if extra_fields:
|
|
266
|
+
for extra_field_path in extra_fields:
|
|
267
|
+
try:
|
|
268
|
+
extra_field_value = dpath.get(
|
|
269
|
+
parent_record, # type: ignore [arg-type]
|
|
270
|
+
extra_field_path,
|
|
271
|
+
)
|
|
272
|
+
self.logger.debug(
|
|
273
|
+
f"Extracted extra_field_path: {extra_field_path} with value: {extra_field_value}"
|
|
274
|
+
)
|
|
275
|
+
except KeyError:
|
|
276
|
+
self.logger.debug(f"Failed to extract extra_field_path: {extra_field_path}")
|
|
277
|
+
extra_field_value = None
|
|
278
|
+
extracted_extra_fields[".".join(extra_field_path)] = extra_field_value
|
|
279
|
+
return extracted_extra_fields
|
|
280
|
+
|
|
281
|
+
def set_initial_state(self, stream_state: StreamState) -> None:
|
|
282
|
+
"""
|
|
283
|
+
Set the state of the parent streams.
|
|
284
|
+
|
|
285
|
+
If the `parent_state` key is missing from `stream_state`, migrate the child stream state to the parent stream's state format.
|
|
286
|
+
This migration applies only to parent streams with incremental dependencies.
|
|
287
|
+
|
|
288
|
+
Args:
|
|
289
|
+
stream_state (StreamState): The state of the streams to be set.
|
|
290
|
+
|
|
291
|
+
Example of state format:
|
|
292
|
+
{
|
|
293
|
+
"parent_state": {
|
|
294
|
+
"parent_stream_name1": {
|
|
295
|
+
"last_updated": "2023-05-27T00:00:00Z"
|
|
296
|
+
},
|
|
297
|
+
"parent_stream_name2": {
|
|
298
|
+
"last_updated": "2023-05-27T00:00:00Z"
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
Example of migrating to parent state format:
|
|
304
|
+
- Initial state:
|
|
305
|
+
{
|
|
306
|
+
"updated_at": "2023-05-27T00:00:00Z"
|
|
307
|
+
}
|
|
308
|
+
- After migration:
|
|
309
|
+
{
|
|
310
|
+
"updated_at": "2023-05-27T00:00:00Z",
|
|
311
|
+
"parent_state": {
|
|
312
|
+
"parent_stream_name": {
|
|
313
|
+
"parent_stream_cursor": "2023-05-27T00:00:00Z"
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
"""
|
|
318
|
+
if not stream_state:
|
|
319
|
+
return
|
|
320
|
+
|
|
321
|
+
parent_state = stream_state.get("parent_state", {})
|
|
322
|
+
|
|
323
|
+
# Set state for each parent stream with an incremental dependency
|
|
324
|
+
for parent_config in self.parent_stream_configs:
|
|
325
|
+
if (
|
|
326
|
+
not parent_state.get(parent_config.stream.name, {})
|
|
327
|
+
and parent_config.incremental_dependency
|
|
328
|
+
):
|
|
329
|
+
# Migrate child state to parent state format
|
|
330
|
+
parent_state = self._migrate_child_state_to_parent_state(stream_state)
|
|
331
|
+
|
|
332
|
+
if parent_config.incremental_dependency:
|
|
333
|
+
parent_config.stream.state = parent_state.get(parent_config.stream.name, {})
|
|
334
|
+
|
|
335
|
+
def _migrate_child_state_to_parent_state(self, stream_state: StreamState) -> StreamState:
|
|
336
|
+
"""
|
|
337
|
+
Migrate the child or global stream state into the parent stream's state format.
|
|
338
|
+
|
|
339
|
+
This method converts the child stream state—or, if present, the global state—into a format that is
|
|
340
|
+
compatible with parent streams that use incremental synchronization. The migration occurs only for
|
|
341
|
+
parent streams with incremental dependencies. It filters out per-partition states and retains only the
|
|
342
|
+
global state in the form {cursor_field: cursor_value}.
|
|
343
|
+
|
|
344
|
+
The method supports multiple input formats:
|
|
345
|
+
- A simple global state, e.g.:
|
|
346
|
+
{"updated_at": "2023-05-27T00:00:00Z"}
|
|
347
|
+
- A state object that contains a "state" key (which is assumed to hold the global state), e.g.:
|
|
348
|
+
{"state": {"updated_at": "2023-05-27T00:00:00Z"}, ...}
|
|
349
|
+
In this case, the migration uses the first value from the "state" dictionary.
|
|
350
|
+
- Any per-partition state formats or other non-simple structures are ignored during migration.
|
|
351
|
+
|
|
352
|
+
Args:
|
|
353
|
+
stream_state (StreamState): The state to migrate. Expected formats include:
|
|
354
|
+
- {"updated_at": "2023-05-27T00:00:00Z"}
|
|
355
|
+
- {"state": {"updated_at": "2023-05-27T00:00:00Z"}, ...}
|
|
356
|
+
(In this format, only the first global state value is used, and per-partition states are ignored.)
|
|
357
|
+
|
|
358
|
+
Returns:
|
|
359
|
+
StreamState: A migrated state for parent streams in the format:
|
|
360
|
+
{
|
|
361
|
+
"parent_stream_name": {"parent_stream_cursor": "2023-05-27T00:00:00Z"}
|
|
362
|
+
}
|
|
363
|
+
where each parent stream with an incremental dependency is assigned its corresponding cursor value.
|
|
364
|
+
|
|
365
|
+
Example:
|
|
366
|
+
Input: {"updated_at": "2023-05-27T00:00:00Z"}
|
|
367
|
+
Output: {
|
|
368
|
+
"parent_stream_name": {"parent_stream_cursor": "2023-05-27T00:00:00Z"}
|
|
369
|
+
}
|
|
370
|
+
"""
|
|
371
|
+
substream_state_values = list(stream_state.values())
|
|
372
|
+
substream_state = substream_state_values[0] if substream_state_values else {}
|
|
373
|
+
|
|
374
|
+
# Ignore per-partition states or invalid formats.
|
|
375
|
+
if isinstance(substream_state, (list, dict)) or len(substream_state_values) != 1:
|
|
376
|
+
# If a global state is present under the key "state", use its first value.
|
|
377
|
+
if "state" in stream_state and isinstance(stream_state["state"], dict):
|
|
378
|
+
substream_state = list(stream_state["state"].values())[0]
|
|
379
|
+
else:
|
|
380
|
+
return {}
|
|
381
|
+
|
|
382
|
+
# Build the parent state for all parent streams with incremental dependencies.
|
|
383
|
+
parent_state = {}
|
|
384
|
+
if substream_state:
|
|
385
|
+
for parent_config in self.parent_stream_configs:
|
|
386
|
+
if parent_config.incremental_dependency:
|
|
387
|
+
parent_state[parent_config.stream.name] = {
|
|
388
|
+
parent_config.stream.cursor_field: substream_state
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
return parent_state
|
|
392
|
+
|
|
393
|
+
def get_stream_state(self) -> Optional[Mapping[str, StreamState]]:
|
|
394
|
+
"""
|
|
395
|
+
Get the state of the parent streams.
|
|
396
|
+
|
|
397
|
+
Returns:
|
|
398
|
+
StreamState: The current state of the parent streams.
|
|
399
|
+
|
|
400
|
+
Example of state format:
|
|
401
|
+
{
|
|
402
|
+
"parent_stream_name1": {
|
|
403
|
+
"last_updated": "2023-05-27T00:00:00Z"
|
|
404
|
+
},
|
|
405
|
+
"parent_stream_name2": {
|
|
406
|
+
"last_updated": "2023-05-27T00:00:00Z"
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
"""
|
|
410
|
+
parent_state = {}
|
|
411
|
+
for parent_config in self.parent_stream_configs:
|
|
412
|
+
if parent_config.incremental_dependency:
|
|
413
|
+
parent_state[parent_config.stream.name] = copy.deepcopy(parent_config.stream.state)
|
|
414
|
+
return parent_state
|
|
415
|
+
|
|
416
|
+
@property
|
|
417
|
+
def logger(self) -> logging.Logger:
|
|
418
|
+
return logging.getLogger("airbyte.SubstreamPartitionRouter")
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
class SafeResponse(requests.Response):
|
|
422
|
+
"""
|
|
423
|
+
A subclass of requests.Response that acts as an interface to migrate parsed child records
|
|
424
|
+
into a response object. This allows seamless interaction with child records as if they
|
|
425
|
+
were original response, ensuring compatibility with methods that expect requests.Response data type.
|
|
426
|
+
"""
|
|
427
|
+
|
|
428
|
+
def __getattr__(self, name: str) -> Any:
|
|
429
|
+
return getattr(requests.Response, name, None)
|
|
430
|
+
|
|
431
|
+
@property
|
|
432
|
+
def content(self) -> Optional[bytes]:
|
|
433
|
+
return super().content
|
|
434
|
+
|
|
435
|
+
@content.setter
|
|
436
|
+
def content(self, value: Union[str, bytes]) -> None:
|
|
437
|
+
self._content = value.encode() if isinstance(value, str) else value
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# AsyncHttpJobRepository sequence diagram
|
|
2
|
+
|
|
3
|
+
- Components marked as optional are not required and can be ignored.
|
|
4
|
+
- if `download_target_requester` is not provided, `download_target_extractor` will get urls from the `polling_response`
|
|
5
|
+
- interpolation_context, e.g. `creation_response` or `polling_response` can be obtained from stream_slice
|
|
6
|
+
|
|
7
|
+
```mermaid
|
|
8
|
+
---
|
|
9
|
+
title: AsyncHttpJobRepository Sequence Diagram
|
|
10
|
+
---
|
|
11
|
+
sequenceDiagram
|
|
12
|
+
participant AsyncHttpJobRepository as AsyncOrchestrator
|
|
13
|
+
participant CreationRequester as creation_requester
|
|
14
|
+
participant PollingRequester as polling_requester
|
|
15
|
+
participant UrlRequester as download_target_requester (Optional)
|
|
16
|
+
participant DownloadRetriever as download_retriever
|
|
17
|
+
participant AbortRequester as abort_requester (Optional)
|
|
18
|
+
participant DeleteRequester as delete_requester (Optional)
|
|
19
|
+
participant Reporting Server as Async Reporting Server
|
|
20
|
+
|
|
21
|
+
AsyncHttpJobRepository ->> CreationRequester: Initiate job creation
|
|
22
|
+
CreationRequester ->> Reporting Server: Create job request
|
|
23
|
+
Reporting Server -->> CreationRequester: Job ID response
|
|
24
|
+
CreationRequester -->> AsyncHttpJobRepository: Job ID
|
|
25
|
+
|
|
26
|
+
loop Poll for job status
|
|
27
|
+
AsyncHttpJobRepository ->> PollingRequester: Check job status
|
|
28
|
+
PollingRequester ->> Reporting Server: Status request (interpolation_context: `creation_response`)
|
|
29
|
+
Reporting Server -->> PollingRequester: Status response
|
|
30
|
+
PollingRequester -->> AsyncHttpJobRepository: Job status
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
alt Status: Ready
|
|
34
|
+
AsyncHttpJobRepository ->> UrlRequester: Request download URLs (if applicable)
|
|
35
|
+
UrlRequester ->> Reporting Server: URL request (interpolation_context: `polling_response`)
|
|
36
|
+
Reporting Server -->> UrlRequester: Download URLs
|
|
37
|
+
UrlRequester -->> AsyncHttpJobRepository: Download URLs
|
|
38
|
+
|
|
39
|
+
AsyncHttpJobRepository ->> DownloadRetriever: Download reports
|
|
40
|
+
DownloadRetriever ->> Reporting Server: Retrieve report data (interpolation_context: `url`)
|
|
41
|
+
Reporting Server -->> DownloadRetriever: Report data
|
|
42
|
+
DownloadRetriever -->> AsyncHttpJobRepository: Report data
|
|
43
|
+
else Status: Failed
|
|
44
|
+
AsyncHttpJobRepository ->> AbortRequester: Send abort request
|
|
45
|
+
AbortRequester ->> Reporting Server: Abort job
|
|
46
|
+
Reporting Server -->> AbortRequester: Abort confirmation
|
|
47
|
+
AbortRequester -->> AsyncHttpJobRepository: Confirmation
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
AsyncHttpJobRepository ->> DeleteRequester: Send delete job request
|
|
51
|
+
DeleteRequester ->> Reporting Server: Delete job
|
|
52
|
+
Reporting Server -->> DeleteRequester: Deletion confirmation
|
|
53
|
+
DeleteRequester -->> AsyncHttpJobRepository: Confirmation
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
```
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester
|
|
6
|
+
from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption
|
|
7
|
+
from airbyte_cdk.sources.declarative.requesters.requester import Requester
|
|
8
|
+
|
|
9
|
+
__all__ = ["HttpRequester", "RequestOption", "Requester"]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import (
|
|
6
|
+
BackoffStrategy,
|
|
7
|
+
)
|
|
8
|
+
from airbyte_cdk.sources.declarative.requesters.error_handlers.composite_error_handler import (
|
|
9
|
+
CompositeErrorHandler,
|
|
10
|
+
)
|
|
11
|
+
from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import (
|
|
12
|
+
DefaultErrorHandler,
|
|
13
|
+
)
|
|
14
|
+
from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler
|
|
15
|
+
from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import (
|
|
16
|
+
HttpResponseFilter,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"BackoffStrategy",
|
|
21
|
+
"CompositeErrorHandler",
|
|
22
|
+
"DefaultErrorHandler",
|
|
23
|
+
"ErrorHandler",
|
|
24
|
+
"HttpResponseFilter",
|
|
25
|
+
]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.constant_backoff_strategy import (
|
|
6
|
+
ConstantBackoffStrategy,
|
|
7
|
+
)
|
|
8
|
+
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import (
|
|
9
|
+
ExponentialBackoffStrategy,
|
|
10
|
+
)
|
|
11
|
+
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.wait_time_from_header_backoff_strategy import (
|
|
12
|
+
WaitTimeFromHeaderBackoffStrategy,
|
|
13
|
+
)
|
|
14
|
+
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.wait_until_time_from_header_backoff_strategy import (
|
|
15
|
+
WaitUntilTimeFromHeaderBackoffStrategy,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"ConstantBackoffStrategy",
|
|
20
|
+
"ExponentialBackoffStrategy",
|
|
21
|
+
"WaitTimeFromHeaderBackoffStrategy",
|
|
22
|
+
"WaitUntilTimeFromHeaderBackoffStrategy",
|
|
23
|
+
]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from dataclasses import InitVar, dataclass
|
|
6
|
+
from typing import Any, Mapping, Optional, Union
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
|
11
|
+
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
|
12
|
+
from airbyte_cdk.sources.types import Config
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class ConstantBackoffStrategy(BackoffStrategy):
|
|
17
|
+
"""
|
|
18
|
+
Backoff strategy with a constant backoff interval
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
backoff_time_in_seconds (float): time to backoff before retrying a retryable request.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
backoff_time_in_seconds: Union[float, InterpolatedString, str]
|
|
25
|
+
parameters: InitVar[Mapping[str, Any]]
|
|
26
|
+
config: Config
|
|
27
|
+
|
|
28
|
+
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
|
29
|
+
if not isinstance(self.backoff_time_in_seconds, InterpolatedString):
|
|
30
|
+
self.backoff_time_in_seconds = str(self.backoff_time_in_seconds)
|
|
31
|
+
if isinstance(self.backoff_time_in_seconds, float):
|
|
32
|
+
self.backoff_time_in_seconds = InterpolatedString.create(
|
|
33
|
+
str(self.backoff_time_in_seconds), parameters=parameters
|
|
34
|
+
)
|
|
35
|
+
else:
|
|
36
|
+
self.backoff_time_in_seconds = InterpolatedString.create(
|
|
37
|
+
self.backoff_time_in_seconds, parameters=parameters
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
def backoff_time(
|
|
41
|
+
self,
|
|
42
|
+
response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
|
|
43
|
+
attempt_count: int,
|
|
44
|
+
) -> Optional[float]:
|
|
45
|
+
return self.backoff_time_in_seconds.eval(self.config) # type: ignore # backoff_time_in_seconds is always cast to an interpolated string
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from dataclasses import InitVar, dataclass
|
|
6
|
+
from typing import Any, Mapping, Optional, Union
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
|
11
|
+
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
|
12
|
+
from airbyte_cdk.sources.types import Config
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class ExponentialBackoffStrategy(BackoffStrategy):
|
|
17
|
+
"""
|
|
18
|
+
Backoff strategy with an exponential backoff interval
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
factor (float): multiplicative factor
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
parameters: InitVar[Mapping[str, Any]]
|
|
25
|
+
config: Config
|
|
26
|
+
factor: Union[float, InterpolatedString, str] = 5
|
|
27
|
+
|
|
28
|
+
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
|
29
|
+
if not isinstance(self.factor, InterpolatedString):
|
|
30
|
+
self.factor = str(self.factor)
|
|
31
|
+
if isinstance(self.factor, float):
|
|
32
|
+
self._factor = InterpolatedString.create(str(self.factor), parameters=parameters)
|
|
33
|
+
else:
|
|
34
|
+
self._factor = InterpolatedString.create(self.factor, parameters=parameters)
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def _retry_factor(self) -> float:
|
|
38
|
+
return self._factor.eval(self.config) # type: ignore # factor is always cast to an interpolated string
|
|
39
|
+
|
|
40
|
+
def backoff_time(
|
|
41
|
+
self,
|
|
42
|
+
response_or_exception: Optional[Union[requests.Response, requests.RequestException]],
|
|
43
|
+
attempt_count: int,
|
|
44
|
+
) -> Optional[float]:
|
|
45
|
+
return self._retry_factor * 2**attempt_count # type: ignore # factor is always cast to an interpolated string
|