airbyte-cdk 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/__init__.py +358 -0
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +5 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +236 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +104 -0
- airbyte_cdk/connector.py +123 -0
- airbyte_cdk/connector_builder/README.md +53 -0
- airbyte_cdk/connector_builder/__init__.py +3 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +121 -0
- airbyte_cdk/connector_builder/main.py +107 -0
- airbyte_cdk/connector_builder/models.py +73 -0
- airbyte_cdk/connector_builder/test_reader/__init__.py +7 -0
- airbyte_cdk/connector_builder/test_reader/helpers.py +689 -0
- airbyte_cdk/connector_builder/test_reader/message_grouper.py +173 -0
- airbyte_cdk/connector_builder/test_reader/reader.py +441 -0
- airbyte_cdk/connector_builder/test_reader/types.py +83 -0
- airbyte_cdk/destinations/__init__.py +8 -0
- airbyte_cdk/destinations/destination.py +154 -0
- airbyte_cdk/destinations/vector_db_based/README.md +37 -0
- airbyte_cdk/destinations/vector_db_based/__init__.py +38 -0
- airbyte_cdk/destinations/vector_db_based/config.py +298 -0
- airbyte_cdk/destinations/vector_db_based/document_processor.py +223 -0
- airbyte_cdk/destinations/vector_db_based/embedder.py +303 -0
- airbyte_cdk/destinations/vector_db_based/indexer.py +78 -0
- airbyte_cdk/destinations/vector_db_based/test_utils.py +63 -0
- airbyte_cdk/destinations/vector_db_based/utils.py +35 -0
- airbyte_cdk/destinations/vector_db_based/writer.py +104 -0
- airbyte_cdk/entrypoint.py +414 -0
- airbyte_cdk/exception_handler.py +56 -0
- airbyte_cdk/logger.py +109 -0
- airbyte_cdk/models/__init__.py +72 -0
- airbyte_cdk/models/airbyte_protocol.py +88 -0
- airbyte_cdk/models/airbyte_protocol_serializers.py +44 -0
- airbyte_cdk/models/well_known_types.py +5 -0
- airbyte_cdk/py.typed +0 -0
- airbyte_cdk/sources/__init__.py +26 -0
- airbyte_cdk/sources/abstract_source.py +326 -0
- airbyte_cdk/sources/concurrent_source/__init__.py +8 -0
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +255 -0
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +165 -0
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +147 -0
- airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py +24 -0
- airbyte_cdk/sources/concurrent_source/stream_thread_exception.py +25 -0
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +115 -0
- airbyte_cdk/sources/config.py +27 -0
- airbyte_cdk/sources/connector_state_manager.py +161 -0
- airbyte_cdk/sources/declarative/__init__.py +3 -0
- airbyte_cdk/sources/declarative/async_job/__init__.py +0 -0
- airbyte_cdk/sources/declarative/async_job/job.py +52 -0
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +525 -0
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +79 -0
- airbyte_cdk/sources/declarative/async_job/repository.py +35 -0
- airbyte_cdk/sources/declarative/async_job/status.py +24 -0
- airbyte_cdk/sources/declarative/async_job/timer.py +39 -0
- airbyte_cdk/sources/declarative/auth/__init__.py +8 -0
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +42 -0
- airbyte_cdk/sources/declarative/auth/jwt.py +197 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +293 -0
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +45 -0
- airbyte_cdk/sources/declarative/auth/token.py +267 -0
- airbyte_cdk/sources/declarative/auth/token_provider.py +82 -0
- airbyte_cdk/sources/declarative/checks/__init__.py +24 -0
- airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +61 -0
- airbyte_cdk/sources/declarative/checks/check_stream.py +56 -0
- airbyte_cdk/sources/declarative/checks/connection_checker.py +35 -0
- airbyte_cdk/sources/declarative/concurrency_level/__init__.py +7 -0
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +50 -0
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +526 -0
- airbyte_cdk/sources/declarative/datetime/__init__.py +3 -0
- airbyte_cdk/sources/declarative/datetime/datetime_parser.py +65 -0
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +118 -0
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +3975 -0
- airbyte_cdk/sources/declarative/declarative_source.py +36 -0
- airbyte_cdk/sources/declarative/declarative_stream.py +241 -0
- airbyte_cdk/sources/declarative/decoders/__init__.py +33 -0
- airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +218 -0
- airbyte_cdk/sources/declarative/decoders/decoder.py +32 -0
- airbyte_cdk/sources/declarative/decoders/decoder_parser.py +30 -0
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +65 -0
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +21 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +39 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +98 -0
- airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py +56 -0
- airbyte_cdk/sources/declarative/exceptions.py +9 -0
- airbyte_cdk/sources/declarative/extractors/__init__.py +21 -0
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +86 -0
- airbyte_cdk/sources/declarative/extractors/http_selector.py +37 -0
- airbyte_cdk/sources/declarative/extractors/record_extractor.py +27 -0
- airbyte_cdk/sources/declarative/extractors/record_filter.py +91 -0
- airbyte_cdk/sources/declarative/extractors/record_selector.py +170 -0
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +176 -0
- airbyte_cdk/sources/declarative/extractors/type_transformer.py +55 -0
- airbyte_cdk/sources/declarative/incremental/__init__.py +37 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +497 -0
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +459 -0
- airbyte_cdk/sources/declarative/incremental/declarative_cursor.py +13 -0
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +357 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +380 -0
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +200 -0
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +122 -0
- airbyte_cdk/sources/declarative/interpolation/__init__.py +9 -0
- airbyte_cdk/sources/declarative/interpolation/filters.py +139 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +66 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +56 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +52 -0
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +79 -0
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +34 -0
- airbyte_cdk/sources/declarative/interpolation/jinja.py +161 -0
- airbyte_cdk/sources/declarative/interpolation/macros.py +191 -0
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +421 -0
- airbyte_cdk/sources/declarative/migrations/__init__.py +0 -0
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +98 -0
- airbyte_cdk/sources/declarative/migrations/state_migration.py +24 -0
- airbyte_cdk/sources/declarative/models/__init__.py +2 -0
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2503 -0
- airbyte_cdk/sources/declarative/parsers/__init__.py +3 -0
- airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py +157 -0
- airbyte_cdk/sources/declarative/parsers/custom_exceptions.py +21 -0
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +172 -0
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +213 -0
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +3407 -0
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +29 -0
- airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py +65 -0
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +176 -0
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +121 -0
- airbyte_cdk/sources/declarative/partition_routers/partition_router.py +62 -0
- airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +63 -0
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +437 -0
- airbyte_cdk/sources/declarative/requesters/README.md +56 -0
- airbyte_cdk/sources/declarative/requesters/__init__.py +9 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/__init__.py +25 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/__init__.py +23 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +45 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +45 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +41 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +70 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +77 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategy.py +17 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +101 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +147 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +40 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +17 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +179 -0
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +350 -0
- airbyte_cdk/sources/declarative/requesters/http_requester.py +433 -0
- airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +21 -0
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +327 -0
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +76 -0
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +65 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +25 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +98 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +102 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +71 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +48 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +66 -0
- airbyte_cdk/sources/declarative/requesters/request_option.py +117 -0
- airbyte_cdk/sources/declarative/requesters/request_options/__init__.py +23 -0
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +92 -0
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +60 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +59 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +68 -0
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +119 -0
- airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +79 -0
- airbyte_cdk/sources/declarative/requesters/request_path.py +15 -0
- airbyte_cdk/sources/declarative/requesters/requester.py +144 -0
- airbyte_cdk/sources/declarative/resolvers/__init__.py +41 -0
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +55 -0
- airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py +136 -0
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +112 -0
- airbyte_cdk/sources/declarative/retrievers/__init__.py +19 -0
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +124 -0
- airbyte_cdk/sources/declarative/retrievers/file_uploader.py +89 -0
- airbyte_cdk/sources/declarative/retrievers/retriever.py +54 -0
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +702 -0
- airbyte_cdk/sources/declarative/schema/__init__.py +25 -0
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +47 -0
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +285 -0
- airbyte_cdk/sources/declarative/schema/inline_schema_loader.py +19 -0
- airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +92 -0
- airbyte_cdk/sources/declarative/schema/schema_loader.py +17 -0
- airbyte_cdk/sources/declarative/spec/__init__.py +7 -0
- airbyte_cdk/sources/declarative/spec/spec.py +48 -0
- airbyte_cdk/sources/declarative/stream_slicers/__init__.py +7 -0
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +93 -0
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +25 -0
- airbyte_cdk/sources/declarative/transformations/__init__.py +17 -0
- airbyte_cdk/sources/declarative/transformations/add_fields.py +146 -0
- airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py +61 -0
- airbyte_cdk/sources/declarative/transformations/flatten_fields.py +52 -0
- airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py +61 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_lower_transformation.py +22 -0
- airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py +68 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +75 -0
- airbyte_cdk/sources/declarative/transformations/transformation.py +37 -0
- airbyte_cdk/sources/declarative/types.py +25 -0
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +67 -0
- airbyte_cdk/sources/file_based/README.md +152 -0
- airbyte_cdk/sources/file_based/__init__.py +24 -0
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +11 -0
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +73 -0
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +149 -0
- airbyte_cdk/sources/file_based/config/__init__.py +0 -0
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +153 -0
- airbyte_cdk/sources/file_based/config/avro_format.py +25 -0
- airbyte_cdk/sources/file_based/config/csv_format.py +210 -0
- airbyte_cdk/sources/file_based/config/excel_format.py +18 -0
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +99 -0
- airbyte_cdk/sources/file_based/config/jsonl_format.py +18 -0
- airbyte_cdk/sources/file_based/config/parquet_format.py +25 -0
- airbyte_cdk/sources/file_based/config/unstructured_format.py +102 -0
- airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py +81 -0
- airbyte_cdk/sources/file_based/discovery_policy/__init__.py +8 -0
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +21 -0
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +33 -0
- airbyte_cdk/sources/file_based/exceptions.py +159 -0
- airbyte_cdk/sources/file_based/file_based_source.py +466 -0
- airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py +123 -0
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +209 -0
- airbyte_cdk/sources/file_based/file_record_data.py +22 -0
- airbyte_cdk/sources/file_based/file_types/__init__.py +37 -0
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +233 -0
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +527 -0
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +196 -0
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +30 -0
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +86 -0
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +145 -0
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +275 -0
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +480 -0
- airbyte_cdk/sources/file_based/remote_file.py +18 -0
- airbyte_cdk/sources/file_based/schema_helpers.py +281 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py +17 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +20 -0
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +52 -0
- airbyte_cdk/sources/file_based/stream/__init__.py +13 -0
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +197 -0
- airbyte_cdk/sources/file_based/stream/concurrent/__init__.py +0 -0
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +343 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py +9 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +59 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +313 -0
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +83 -0
- airbyte_cdk/sources/file_based/stream/cursor/__init__.py +4 -0
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +66 -0
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +149 -0
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +396 -0
- airbyte_cdk/sources/file_based/stream/identities_stream.py +49 -0
- airbyte_cdk/sources/file_based/stream/permissions_file_based_stream.py +92 -0
- airbyte_cdk/sources/file_based/types.py +10 -0
- airbyte_cdk/sources/http_config.py +10 -0
- airbyte_cdk/sources/http_logger.py +55 -0
- airbyte_cdk/sources/message/__init__.py +19 -0
- airbyte_cdk/sources/message/repository.py +137 -0
- airbyte_cdk/sources/source.py +95 -0
- airbyte_cdk/sources/specs/transfer_modes.py +26 -0
- airbyte_cdk/sources/streams/__init__.py +8 -0
- airbyte_cdk/sources/streams/availability_strategy.py +84 -0
- airbyte_cdk/sources/streams/call_rate.py +704 -0
- airbyte_cdk/sources/streams/checkpoint/__init__.py +26 -0
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +335 -0
- airbyte_cdk/sources/streams/checkpoint/cursor.py +77 -0
- airbyte_cdk/sources/streams/checkpoint/per_partition_key_serializer.py +22 -0
- airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py +51 -0
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +110 -0
- airbyte_cdk/sources/streams/concurrent/README.md +7 -0
- airbyte_cdk/sources/streams/concurrent/__init__.py +3 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +96 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py +37 -0
- airbyte_cdk/sources/streams/concurrent/adapters.py +397 -0
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +94 -0
- airbyte_cdk/sources/streams/concurrent/clamping.py +99 -0
- airbyte_cdk/sources/streams/concurrent/cursor.py +481 -0
- airbyte_cdk/sources/streams/concurrent/cursor_types.py +32 -0
- airbyte_cdk/sources/streams/concurrent/default_stream.py +102 -0
- airbyte_cdk/sources/streams/concurrent/exceptions.py +18 -0
- airbyte_cdk/sources/streams/concurrent/helpers.py +42 -0
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +64 -0
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +45 -0
- airbyte_cdk/sources/streams/concurrent/partitions/__init__.py +3 -0
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +48 -0
- airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py +18 -0
- airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py +21 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +38 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py +0 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +182 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +223 -0
- airbyte_cdk/sources/streams/concurrent/state_converters/incrementing_count_stream_state_converter.py +92 -0
- airbyte_cdk/sources/streams/core.py +703 -0
- airbyte_cdk/sources/streams/http/__init__.py +10 -0
- airbyte_cdk/sources/streams/http/availability_strategy.py +54 -0
- airbyte_cdk/sources/streams/http/error_handlers/__init__.py +22 -0
- airbyte_cdk/sources/streams/http/error_handlers/backoff_strategy.py +28 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_backoff_strategy.py +17 -0
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +86 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +42 -0
- airbyte_cdk/sources/streams/http/error_handlers/error_message_parser.py +19 -0
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +110 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +52 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +65 -0
- airbyte_cdk/sources/streams/http/exceptions.py +61 -0
- airbyte_cdk/sources/streams/http/http.py +673 -0
- airbyte_cdk/sources/streams/http/http_client.py +531 -0
- airbyte_cdk/sources/streams/http/rate_limiting.py +158 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py +14 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +479 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +34 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +436 -0
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +83 -0
- airbyte_cdk/sources/streams/permissions/identities_stream.py +75 -0
- airbyte_cdk/sources/streams/utils/__init__.py +3 -0
- airbyte_cdk/sources/types.py +169 -0
- airbyte_cdk/sources/utils/__init__.py +7 -0
- airbyte_cdk/sources/utils/casing.py +12 -0
- airbyte_cdk/sources/utils/files_directory.py +15 -0
- airbyte_cdk/sources/utils/record_helper.py +53 -0
- airbyte_cdk/sources/utils/schema_helpers.py +230 -0
- airbyte_cdk/sources/utils/slice_logger.py +57 -0
- airbyte_cdk/sources/utils/transform.py +277 -0
- airbyte_cdk/sources/utils/types.py +7 -0
- airbyte_cdk/sql/__init__.py +0 -0
- airbyte_cdk/sql/_util/__init__.py +0 -0
- airbyte_cdk/sql/_util/hashing.py +34 -0
- airbyte_cdk/sql/_util/name_normalizers.py +92 -0
- airbyte_cdk/sql/constants.py +32 -0
- airbyte_cdk/sql/exceptions.py +235 -0
- airbyte_cdk/sql/secrets.py +123 -0
- airbyte_cdk/sql/shared/__init__.py +15 -0
- airbyte_cdk/sql/shared/catalog_providers.py +145 -0
- airbyte_cdk/sql/shared/sql_processor.py +786 -0
- airbyte_cdk/sql/types.py +160 -0
- airbyte_cdk/test/__init__.py +7 -0
- airbyte_cdk/test/catalog_builder.py +81 -0
- airbyte_cdk/test/entrypoint_wrapper.py +250 -0
- airbyte_cdk/test/mock_http/__init__.py +6 -0
- airbyte_cdk/test/mock_http/matcher.py +41 -0
- airbyte_cdk/test/mock_http/mocker.py +185 -0
- airbyte_cdk/test/mock_http/request.py +103 -0
- airbyte_cdk/test/mock_http/response.py +28 -0
- airbyte_cdk/test/mock_http/response_builder.py +237 -0
- airbyte_cdk/test/state_builder.py +33 -0
- airbyte_cdk/test/utils/__init__.py +1 -0
- airbyte_cdk/test/utils/data.py +24 -0
- airbyte_cdk/test/utils/http_mocking.py +16 -0
- airbyte_cdk/test/utils/manifest_only_fixtures.py +59 -0
- airbyte_cdk/test/utils/reading.py +26 -0
- airbyte_cdk/utils/__init__.py +10 -0
- airbyte_cdk/utils/airbyte_secrets_utils.py +80 -0
- airbyte_cdk/utils/analytics_message.py +25 -0
- airbyte_cdk/utils/constants.py +5 -0
- airbyte_cdk/utils/datetime_format_inferrer.py +94 -0
- airbyte_cdk/utils/datetime_helpers.py +499 -0
- airbyte_cdk/utils/event_timing.py +85 -0
- airbyte_cdk/utils/is_cloud_environment.py +18 -0
- airbyte_cdk/utils/mapping_helpers.py +162 -0
- airbyte_cdk/utils/message_utils.py +26 -0
- airbyte_cdk/utils/oneof_option_config.py +33 -0
- airbyte_cdk/utils/print_buffer.py +75 -0
- airbyte_cdk/utils/schema_inferrer.py +270 -0
- airbyte_cdk/utils/slice_hasher.py +37 -0
- airbyte_cdk/utils/spec_schema_transformations.py +26 -0
- airbyte_cdk/utils/stream_status_utils.py +43 -0
- airbyte_cdk/utils/traced_exception.py +145 -0
- airbyte_cdk-0.0.0.dev0.dist-info/LICENSE.txt +19 -0
- airbyte_cdk-0.0.0.dev0.dist-info/LICENSE_SHORT +1 -0
- airbyte_cdk-0.0.0.dev0.dist-info/METADATA +111 -0
- airbyte_cdk-0.0.0.dev0.dist-info/RECORD +368 -0
- airbyte_cdk-0.0.0.dev0.dist-info/WHEEL +4 -0
- airbyte_cdk-0.0.0.dev0.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from abc import abstractmethod
|
|
7
|
+
from typing import Any, Mapping, Tuple
|
|
8
|
+
|
|
9
|
+
from airbyte_cdk.sources.abstract_source import AbstractSource
|
|
10
|
+
from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class DeclarativeSource(AbstractSource):
|
|
14
|
+
"""
|
|
15
|
+
Base class for declarative Source. Concrete sources need to define the connection_checker to use
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
@abstractmethod
|
|
20
|
+
def connection_checker(self) -> ConnectionChecker:
|
|
21
|
+
"""Returns the ConnectionChecker to use for the `check` operation"""
|
|
22
|
+
|
|
23
|
+
def check_connection(
|
|
24
|
+
self, logger: logging.Logger, config: Mapping[str, Any]
|
|
25
|
+
) -> Tuple[bool, Any]:
|
|
26
|
+
"""
|
|
27
|
+
:param logger: The source logger
|
|
28
|
+
:param config: The user-provided configuration as specified by the source's spec.
|
|
29
|
+
This usually contains information required to check connection e.g. tokens, secrets and keys etc.
|
|
30
|
+
:return: A tuple of (boolean, error). If boolean is true, then the connection check is successful
|
|
31
|
+
and we can connect to the underlying data source using the provided configuration.
|
|
32
|
+
Otherwise, the input config cannot be used to connect to the underlying data source,
|
|
33
|
+
and the "error" object should describe what went wrong.
|
|
34
|
+
The error object will be cast to string to display the problem to the user.
|
|
35
|
+
"""
|
|
36
|
+
return self.connection_checker.check_connection(self, logger, config)
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
import logging
|
|
5
|
+
from dataclasses import InitVar, dataclass, field
|
|
6
|
+
from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
|
|
7
|
+
|
|
8
|
+
from airbyte_cdk.models import SyncMode
|
|
9
|
+
from airbyte_cdk.sources.declarative.incremental import (
|
|
10
|
+
GlobalSubstreamCursor,
|
|
11
|
+
PerPartitionCursor,
|
|
12
|
+
PerPartitionWithGlobalCursor,
|
|
13
|
+
)
|
|
14
|
+
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
|
15
|
+
from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
|
|
16
|
+
from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever
|
|
17
|
+
from airbyte_cdk.sources.declarative.retrievers.async_retriever import AsyncRetriever
|
|
18
|
+
from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
|
|
19
|
+
from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader
|
|
20
|
+
from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
|
|
21
|
+
from airbyte_cdk.sources.streams.checkpoint import (
|
|
22
|
+
CheckpointMode,
|
|
23
|
+
CheckpointReader,
|
|
24
|
+
Cursor,
|
|
25
|
+
CursorBasedCheckpointReader,
|
|
26
|
+
)
|
|
27
|
+
from airbyte_cdk.sources.streams.core import Stream
|
|
28
|
+
from airbyte_cdk.sources.types import Config, StreamSlice
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class DeclarativeStream(Stream):
|
|
33
|
+
"""
|
|
34
|
+
DeclarativeStream is a Stream that delegates most of its logic to its schema_load and retriever
|
|
35
|
+
|
|
36
|
+
Attributes:
|
|
37
|
+
name (str): stream name
|
|
38
|
+
primary_key (Optional[Union[str, List[str], List[List[str]]]]): the primary key of the stream
|
|
39
|
+
schema_loader (SchemaLoader): The schema loader
|
|
40
|
+
retriever (Retriever): The retriever
|
|
41
|
+
config (Config): The user-provided configuration as specified by the source's spec
|
|
42
|
+
stream_cursor_field (Optional[Union[InterpolatedString, str]]): The cursor field
|
|
43
|
+
stream. Transformations are applied in the order in which they are defined.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
retriever: Retriever
|
|
47
|
+
config: Config
|
|
48
|
+
parameters: InitVar[Mapping[str, Any]]
|
|
49
|
+
name: str
|
|
50
|
+
primary_key: Optional[Union[str, List[str], List[List[str]]]]
|
|
51
|
+
state_migrations: List[StateMigration] = field(repr=True, default_factory=list)
|
|
52
|
+
schema_loader: Optional[SchemaLoader] = None
|
|
53
|
+
_name: str = field(init=False, repr=False, default="")
|
|
54
|
+
_primary_key: str = field(init=False, repr=False, default="")
|
|
55
|
+
stream_cursor_field: Optional[Union[InterpolatedString, str]] = None
|
|
56
|
+
|
|
57
|
+
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
|
58
|
+
self._stream_cursor_field = (
|
|
59
|
+
InterpolatedString.create(self.stream_cursor_field, parameters=parameters)
|
|
60
|
+
if isinstance(self.stream_cursor_field, str)
|
|
61
|
+
else self.stream_cursor_field
|
|
62
|
+
)
|
|
63
|
+
self._schema_loader = (
|
|
64
|
+
self.schema_loader
|
|
65
|
+
if self.schema_loader
|
|
66
|
+
else DefaultSchemaLoader(config=self.config, parameters=parameters)
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
@property # type: ignore
|
|
70
|
+
def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]:
|
|
71
|
+
return self._primary_key
|
|
72
|
+
|
|
73
|
+
@primary_key.setter
|
|
74
|
+
def primary_key(self, value: str) -> None:
|
|
75
|
+
if not isinstance(value, property):
|
|
76
|
+
self._primary_key = value
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def exit_on_rate_limit(self) -> bool:
|
|
80
|
+
if isinstance(self.retriever, AsyncRetriever):
|
|
81
|
+
return self.retriever.exit_on_rate_limit
|
|
82
|
+
|
|
83
|
+
return self.retriever.requester.exit_on_rate_limit # type: ignore # abstract Retriever class has not requester attribute
|
|
84
|
+
|
|
85
|
+
@exit_on_rate_limit.setter
|
|
86
|
+
def exit_on_rate_limit(self, value: bool) -> None:
|
|
87
|
+
if isinstance(self.retriever, AsyncRetriever):
|
|
88
|
+
self.retriever.exit_on_rate_limit = value
|
|
89
|
+
else:
|
|
90
|
+
self.retriever.requester.exit_on_rate_limit = value # type: ignore[attr-defined]
|
|
91
|
+
|
|
92
|
+
@property # type: ignore
|
|
93
|
+
def name(self) -> str:
|
|
94
|
+
"""
|
|
95
|
+
:return: Stream name. By default this is the implementing class name, but it can be overridden as needed.
|
|
96
|
+
"""
|
|
97
|
+
return self._name
|
|
98
|
+
|
|
99
|
+
@name.setter
|
|
100
|
+
def name(self, value: str) -> None:
|
|
101
|
+
if not isinstance(value, property):
|
|
102
|
+
self._name = value
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def state(self) -> MutableMapping[str, Any]:
|
|
106
|
+
return self.retriever.state # type: ignore
|
|
107
|
+
|
|
108
|
+
@state.setter
|
|
109
|
+
def state(self, value: MutableMapping[str, Any]) -> None:
|
|
110
|
+
"""State setter, accept state serialized by state getter."""
|
|
111
|
+
state: Mapping[str, Any] = value
|
|
112
|
+
if self.state_migrations:
|
|
113
|
+
for migration in self.state_migrations:
|
|
114
|
+
if migration.should_migrate(state):
|
|
115
|
+
state = migration.migrate(state)
|
|
116
|
+
self.retriever.state = state
|
|
117
|
+
|
|
118
|
+
def get_updated_state(
|
|
119
|
+
self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]
|
|
120
|
+
) -> MutableMapping[str, Any]:
|
|
121
|
+
return self.state
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def cursor_field(self) -> Union[str, List[str]]:
|
|
125
|
+
"""
|
|
126
|
+
Override to return the default cursor field used by this stream e.g: an API entity might always use created_at as the cursor field.
|
|
127
|
+
:return: The name of the field used as a cursor. If the cursor is nested, return an array consisting of the path to the cursor.
|
|
128
|
+
"""
|
|
129
|
+
cursor = self._stream_cursor_field.eval(self.config) # type: ignore # _stream_cursor_field is always cast to interpolated string
|
|
130
|
+
return cursor if cursor else []
|
|
131
|
+
|
|
132
|
+
@property
|
|
133
|
+
def is_resumable(self) -> bool:
|
|
134
|
+
# Declarative sources always implement state getter/setter, but whether it supports checkpointing is based on
|
|
135
|
+
# if the retriever has a cursor defined.
|
|
136
|
+
return self.retriever.cursor is not None if hasattr(self.retriever, "cursor") else False
|
|
137
|
+
|
|
138
|
+
def read_records(
|
|
139
|
+
self,
|
|
140
|
+
sync_mode: SyncMode,
|
|
141
|
+
cursor_field: Optional[List[str]] = None,
|
|
142
|
+
stream_slice: Optional[Mapping[str, Any]] = None,
|
|
143
|
+
stream_state: Optional[Mapping[str, Any]] = None,
|
|
144
|
+
) -> Iterable[Mapping[str, Any]]:
|
|
145
|
+
"""
|
|
146
|
+
:param: stream_state We knowingly avoid using stream_state as we want cursors to manage their own state.
|
|
147
|
+
"""
|
|
148
|
+
if stream_slice is None or (
|
|
149
|
+
not isinstance(stream_slice, StreamSlice) and stream_slice == {}
|
|
150
|
+
):
|
|
151
|
+
# As the parameter is Optional, many would just call `read_records(sync_mode)` during testing without specifying the field
|
|
152
|
+
# As part of the declarative model without custom components, this should never happen as the CDK would wire up a
|
|
153
|
+
# SinglePartitionRouter that would create this StreamSlice properly
|
|
154
|
+
# As part of the declarative model with custom components, a user that would return a `None` slice would now have the default
|
|
155
|
+
# empty slice which seems to make sense.
|
|
156
|
+
stream_slice = StreamSlice(partition={}, cursor_slice={})
|
|
157
|
+
if not isinstance(stream_slice, StreamSlice):
|
|
158
|
+
raise ValueError(
|
|
159
|
+
f"DeclarativeStream does not support stream_slices that are not StreamSlice. Got {stream_slice}"
|
|
160
|
+
)
|
|
161
|
+
yield from self.retriever.read_records(self.get_json_schema(), stream_slice) # type: ignore # records are of the correct type
|
|
162
|
+
|
|
163
|
+
def get_json_schema(self) -> Mapping[str, Any]: # type: ignore
|
|
164
|
+
"""
|
|
165
|
+
:return: A dict of the JSON schema representing this stream.
|
|
166
|
+
|
|
167
|
+
The default implementation of this method looks for a JSONSchema file with the same name as this stream's "name" property.
|
|
168
|
+
Override as needed.
|
|
169
|
+
"""
|
|
170
|
+
return self._schema_loader.get_json_schema()
|
|
171
|
+
|
|
172
|
+
def stream_slices(
|
|
173
|
+
self,
|
|
174
|
+
*,
|
|
175
|
+
sync_mode: SyncMode,
|
|
176
|
+
cursor_field: Optional[List[str]] = None,
|
|
177
|
+
stream_state: Optional[Mapping[str, Any]] = None,
|
|
178
|
+
) -> Iterable[Optional[StreamSlice]]:
|
|
179
|
+
"""
|
|
180
|
+
Override to define the slices for this stream. See the stream slicing section of the docs for more information.
|
|
181
|
+
|
|
182
|
+
:param sync_mode:
|
|
183
|
+
:param cursor_field:
|
|
184
|
+
:param stream_state: we knowingly avoid using stream_state as we want cursors to manage their own state
|
|
185
|
+
:return:
|
|
186
|
+
"""
|
|
187
|
+
return self.retriever.stream_slices()
|
|
188
|
+
|
|
189
|
+
@property
|
|
190
|
+
def state_checkpoint_interval(self) -> Optional[int]:
|
|
191
|
+
"""
|
|
192
|
+
We explicitly disable checkpointing here. There are a couple reasons for that and not all are documented here but:
|
|
193
|
+
* In the case where records are not ordered, the granularity of what is ordered is the slice. Therefore, we will only update the
|
|
194
|
+
cursor value once at the end of every slice.
|
|
195
|
+
* Updating the state once every record would generate issues for data feed stop conditions or semi-incremental syncs where the
|
|
196
|
+
important state is the one at the beginning of the slice
|
|
197
|
+
"""
|
|
198
|
+
return None
|
|
199
|
+
|
|
200
|
+
def get_cursor(self) -> Optional[Cursor]:
|
|
201
|
+
if self.retriever and isinstance(self.retriever, SimpleRetriever):
|
|
202
|
+
return self.retriever.cursor
|
|
203
|
+
return None
|
|
204
|
+
|
|
205
|
+
def _get_checkpoint_reader(
|
|
206
|
+
self,
|
|
207
|
+
logger: logging.Logger,
|
|
208
|
+
cursor_field: Optional[List[str]],
|
|
209
|
+
sync_mode: SyncMode,
|
|
210
|
+
stream_state: MutableMapping[str, Any],
|
|
211
|
+
) -> CheckpointReader:
|
|
212
|
+
"""
|
|
213
|
+
This method is overridden to prevent issues with stream slice classification for incremental streams that have parent streams.
|
|
214
|
+
|
|
215
|
+
The classification logic, when used with `itertools.tee`, creates a copy of the stream slices. When `stream_slices` is called
|
|
216
|
+
the second time, the parent records generated during the classification phase are lost. This occurs because `itertools.tee`
|
|
217
|
+
only buffers the results, meaning the logic in `simple_retriever` that observes and updates the cursor isn't executed again.
|
|
218
|
+
|
|
219
|
+
By overriding this method, we ensure that the stream slices are processed correctly and parent records are not lost,
|
|
220
|
+
allowing the cursor to function as expected.
|
|
221
|
+
"""
|
|
222
|
+
mappings_or_slices = self.stream_slices(
|
|
223
|
+
cursor_field=cursor_field,
|
|
224
|
+
sync_mode=sync_mode, # todo: change this interface to no longer rely on sync_mode for behavior
|
|
225
|
+
stream_state=stream_state,
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
cursor = self.get_cursor()
|
|
229
|
+
checkpoint_mode = self._checkpoint_mode
|
|
230
|
+
|
|
231
|
+
if isinstance(
|
|
232
|
+
cursor, (GlobalSubstreamCursor, PerPartitionCursor, PerPartitionWithGlobalCursor)
|
|
233
|
+
):
|
|
234
|
+
self.has_multiple_slices = True
|
|
235
|
+
return CursorBasedCheckpointReader(
|
|
236
|
+
stream_slices=mappings_or_slices,
|
|
237
|
+
cursor=cursor,
|
|
238
|
+
read_state_from_cursor=checkpoint_mode == CheckpointMode.RESUMABLE_FULL_REFRESH,
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
return super()._get_checkpoint_reader(logger, cursor_field, sync_mode, stream_state)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import (
|
|
6
|
+
CompositeRawDecoder,
|
|
7
|
+
GzipParser,
|
|
8
|
+
JsonParser,
|
|
9
|
+
Parser,
|
|
10
|
+
)
|
|
11
|
+
from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
|
|
12
|
+
from airbyte_cdk.sources.declarative.decoders.json_decoder import (
|
|
13
|
+
IterableDecoder,
|
|
14
|
+
JsonDecoder,
|
|
15
|
+
)
|
|
16
|
+
from airbyte_cdk.sources.declarative.decoders.noop_decoder import NoopDecoder
|
|
17
|
+
from airbyte_cdk.sources.declarative.decoders.pagination_decoder_decorator import (
|
|
18
|
+
PaginationDecoderDecorator,
|
|
19
|
+
)
|
|
20
|
+
from airbyte_cdk.sources.declarative.decoders.xml_decoder import XmlDecoder
|
|
21
|
+
from airbyte_cdk.sources.declarative.decoders.zipfile_decoder import ZipfileDecoder
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"Decoder",
|
|
25
|
+
"CompositeRawDecoder",
|
|
26
|
+
"JsonDecoder",
|
|
27
|
+
"JsonParser",
|
|
28
|
+
"IterableDecoder",
|
|
29
|
+
"NoopDecoder",
|
|
30
|
+
"PaginationDecoderDecorator",
|
|
31
|
+
"XmlDecoder",
|
|
32
|
+
"ZipfileDecoder",
|
|
33
|
+
]
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import gzip
|
|
7
|
+
import io
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from io import BufferedIOBase, TextIOWrapper
|
|
12
|
+
from typing import Any, Optional
|
|
13
|
+
|
|
14
|
+
import orjson
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
from airbyte_cdk.models import FailureType
|
|
18
|
+
from airbyte_cdk.sources.declarative.decoders.decoder import DECODER_OUTPUT_TYPE, Decoder
|
|
19
|
+
from airbyte_cdk.sources.declarative.decoders.decoder_parser import (
|
|
20
|
+
PARSER_OUTPUT_TYPE,
|
|
21
|
+
PARSERS_BY_HEADER_TYPE,
|
|
22
|
+
PARSERS_TYPE,
|
|
23
|
+
Parser,
|
|
24
|
+
)
|
|
25
|
+
from airbyte_cdk.utils import AirbyteTracedException
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger("airbyte")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class GzipParser(Parser):
|
|
32
|
+
inner_parser: Parser
|
|
33
|
+
|
|
34
|
+
def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
|
|
35
|
+
"""
|
|
36
|
+
Decompress gzipped bytes and pass decompressed data to the inner parser.
|
|
37
|
+
|
|
38
|
+
IMPORTANT:
|
|
39
|
+
- If the data is not gzipped, reset the pointer and pass the data to the inner parser as is.
|
|
40
|
+
|
|
41
|
+
Note:
|
|
42
|
+
- The data is not decoded by default.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
with gzip.GzipFile(fileobj=data, mode="rb") as gzipobj:
|
|
46
|
+
yield from self.inner_parser.parse(gzipobj)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class JsonParser(Parser):
|
|
51
|
+
encoding: str = "utf-8"
|
|
52
|
+
|
|
53
|
+
def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
|
|
54
|
+
"""
|
|
55
|
+
Attempts to deserialize data using orjson library. As an extra layer of safety we fallback on the json library to deserialize the data.
|
|
56
|
+
"""
|
|
57
|
+
raw_data = data.read()
|
|
58
|
+
body_json = self._parse_orjson(raw_data) or self._parse_json(raw_data)
|
|
59
|
+
|
|
60
|
+
if body_json is None:
|
|
61
|
+
raise AirbyteTracedException(
|
|
62
|
+
message="Response JSON data failed to be parsed. See logs for more information.",
|
|
63
|
+
internal_message=f"Response JSON data failed to be parsed.",
|
|
64
|
+
failure_type=FailureType.system_error,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
if isinstance(body_json, list):
|
|
68
|
+
yield from body_json
|
|
69
|
+
else:
|
|
70
|
+
yield from [body_json]
|
|
71
|
+
|
|
72
|
+
def _parse_orjson(self, raw_data: bytes) -> Optional[Any]:
|
|
73
|
+
try:
|
|
74
|
+
return orjson.loads(raw_data.decode(self.encoding))
|
|
75
|
+
except Exception as exc:
|
|
76
|
+
logger.debug(
|
|
77
|
+
f"Failed to parse JSON data using orjson library. Falling back to json library. {exc}"
|
|
78
|
+
)
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
def _parse_json(self, raw_data: bytes) -> Optional[Any]:
|
|
82
|
+
try:
|
|
83
|
+
return json.loads(raw_data.decode(self.encoding))
|
|
84
|
+
except Exception as exc:
|
|
85
|
+
logger.error(f"Failed to parse JSON data using json library. {exc}")
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass
|
|
90
|
+
class JsonLineParser(Parser):
|
|
91
|
+
encoding: Optional[str] = "utf-8"
|
|
92
|
+
|
|
93
|
+
def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
|
|
94
|
+
for line in data:
|
|
95
|
+
try:
|
|
96
|
+
yield json.loads(line.decode(encoding=self.encoding or "utf-8"))
|
|
97
|
+
except json.JSONDecodeError as e:
|
|
98
|
+
logger.warning(f"Cannot decode/parse line {line!r} as JSON, error: {e}")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclass
|
|
102
|
+
class CsvParser(Parser):
|
|
103
|
+
# TODO: migrate implementation to re-use file-base classes
|
|
104
|
+
encoding: Optional[str] = "utf-8"
|
|
105
|
+
delimiter: Optional[str] = ","
|
|
106
|
+
|
|
107
|
+
def _get_delimiter(self) -> Optional[str]:
|
|
108
|
+
"""
|
|
109
|
+
Get delimiter from the configuration. Check for the escape character and decode it.
|
|
110
|
+
"""
|
|
111
|
+
if self.delimiter is not None:
|
|
112
|
+
if self.delimiter.startswith("\\"):
|
|
113
|
+
self.delimiter = self.delimiter.encode("utf-8").decode("unicode_escape")
|
|
114
|
+
|
|
115
|
+
return self.delimiter
|
|
116
|
+
|
|
117
|
+
def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
|
|
118
|
+
"""
|
|
119
|
+
Parse CSV data from decompressed bytes.
|
|
120
|
+
"""
|
|
121
|
+
text_data = TextIOWrapper(data, encoding=self.encoding) # type: ignore
|
|
122
|
+
reader = csv.DictReader(text_data, delimiter=self._get_delimiter() or ",")
|
|
123
|
+
for row in reader:
|
|
124
|
+
yield row
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class CompositeRawDecoder(Decoder):
|
|
128
|
+
"""
|
|
129
|
+
Decoder strategy to transform a requests.Response into a PARSER_OUTPUT_TYPE
|
|
130
|
+
passed response.raw to parser(s).
|
|
131
|
+
|
|
132
|
+
Note: response.raw is not decoded/decompressed by default. Parsers should be instantiated recursively.
|
|
133
|
+
|
|
134
|
+
Example:
|
|
135
|
+
composite_raw_decoder = CompositeRawDecoder(
|
|
136
|
+
parser=GzipParser(
|
|
137
|
+
inner_parser=JsonLineParser(encoding="iso-8859-1")
|
|
138
|
+
)
|
|
139
|
+
)
|
|
140
|
+
"""
|
|
141
|
+
|
|
142
|
+
def __init__(
|
|
143
|
+
self,
|
|
144
|
+
parser: Parser,
|
|
145
|
+
stream_response: bool = True,
|
|
146
|
+
parsers_by_header: PARSERS_BY_HEADER_TYPE = None,
|
|
147
|
+
) -> None:
|
|
148
|
+
# since we moved from using `dataclass` to `__init__` method,
|
|
149
|
+
# we need to keep using the `parser` to be able to resolve the depenencies
|
|
150
|
+
# between the parsers correctly.
|
|
151
|
+
self.parser = parser
|
|
152
|
+
|
|
153
|
+
self._parsers_by_header = parsers_by_header if parsers_by_header else {}
|
|
154
|
+
self._stream_response = stream_response
|
|
155
|
+
|
|
156
|
+
@classmethod
|
|
157
|
+
def by_headers(
|
|
158
|
+
cls,
|
|
159
|
+
parsers: PARSERS_TYPE,
|
|
160
|
+
stream_response: bool,
|
|
161
|
+
fallback_parser: Parser,
|
|
162
|
+
) -> "CompositeRawDecoder":
|
|
163
|
+
"""
|
|
164
|
+
Create a CompositeRawDecoder instance based on header values.
|
|
165
|
+
|
|
166
|
+
Args:
|
|
167
|
+
parsers (PARSERS_TYPE): A list of tuples where each tuple contains headers, header values, and a parser.
|
|
168
|
+
stream_response (bool): A flag indicating whether the response should be streamed.
|
|
169
|
+
fallback_parser (Parser): A parser to use if no matching header is found.
|
|
170
|
+
|
|
171
|
+
Returns:
|
|
172
|
+
CompositeRawDecoder: An instance of CompositeRawDecoder configured with the provided parsers.
|
|
173
|
+
"""
|
|
174
|
+
parsers_by_header = {}
|
|
175
|
+
for headers, header_values, parser in parsers:
|
|
176
|
+
for header in headers:
|
|
177
|
+
parsers_by_header[header] = {header_value: parser for header_value in header_values}
|
|
178
|
+
return cls(fallback_parser, stream_response, parsers_by_header)
|
|
179
|
+
|
|
180
|
+
def is_stream_response(self) -> bool:
|
|
181
|
+
return self._stream_response
|
|
182
|
+
|
|
183
|
+
def decode(self, response: requests.Response) -> DECODER_OUTPUT_TYPE:
|
|
184
|
+
parser = self._select_parser(response)
|
|
185
|
+
if self.is_stream_response():
|
|
186
|
+
# urllib mentions that some interfaces don't play nice with auto_close
|
|
187
|
+
# More info here: https://urllib3.readthedocs.io/en/stable/user-guide.html#using-io-wrappers-with-response-content
|
|
188
|
+
# We have indeed observed some issues with CSV parsing.
|
|
189
|
+
# Hence, we will manage the closing of the file ourselves until we find a better solution.
|
|
190
|
+
response.raw.auto_close = False
|
|
191
|
+
yield from parser.parse(
|
|
192
|
+
data=response.raw, # type: ignore[arg-type]
|
|
193
|
+
)
|
|
194
|
+
response.raw.close()
|
|
195
|
+
else:
|
|
196
|
+
yield from parser.parse(data=io.BytesIO(response.content))
|
|
197
|
+
|
|
198
|
+
def _select_parser(self, response: requests.Response) -> Parser:
|
|
199
|
+
"""
|
|
200
|
+
Selects the appropriate parser based on the response headers.
|
|
201
|
+
|
|
202
|
+
This method iterates through the `_parsers_by_header` dictionary to find a matching parser
|
|
203
|
+
based on the headers in the response. If a matching header and header value are found,
|
|
204
|
+
the corresponding parser is returned. If no match is found, the default parser is returned.
|
|
205
|
+
|
|
206
|
+
Args:
|
|
207
|
+
response (requests.Response): The HTTP response object containing headers to check.
|
|
208
|
+
|
|
209
|
+
Returns:
|
|
210
|
+
Parser: The parser corresponding to the matched header value, or the default parser if no match is found.
|
|
211
|
+
"""
|
|
212
|
+
for header, parser_by_header_value in self._parsers_by_header.items():
|
|
213
|
+
if (
|
|
214
|
+
header in response.headers
|
|
215
|
+
and response.headers[header] in parser_by_header_value.keys()
|
|
216
|
+
):
|
|
217
|
+
return parser_by_header_value[response.headers[header]]
|
|
218
|
+
return self.parser
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from abc import abstractmethod
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Any, Generator, MutableMapping
|
|
8
|
+
|
|
9
|
+
import requests
|
|
10
|
+
|
|
11
|
+
DECODER_OUTPUT_TYPE = Generator[MutableMapping[str, Any], None, None]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class Decoder:
|
|
16
|
+
"""
|
|
17
|
+
Decoder strategy to transform a requests.Response into a Mapping[str, Any]
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def is_stream_response(self) -> bool:
|
|
22
|
+
"""
|
|
23
|
+
Set to True if you'd like to use stream=True option in http requester
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
@abstractmethod
|
|
27
|
+
def decode(self, response: requests.Response) -> DECODER_OUTPUT_TYPE:
|
|
28
|
+
"""
|
|
29
|
+
Decodes a requests.Response into a Mapping[str, Any] or an array
|
|
30
|
+
:param response: the response to decode
|
|
31
|
+
:return: Generator of Mapping describing the response
|
|
32
|
+
"""
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from io import BufferedIOBase
|
|
10
|
+
from typing import Any, Dict, Generator, List, MutableMapping, Optional, Set, Tuple
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger("airbyte")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
PARSER_OUTPUT_TYPE = Generator[MutableMapping[str, Any], None, None]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class Parser(ABC):
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
|
|
22
|
+
"""
|
|
23
|
+
Parse data and yield dictionaries.
|
|
24
|
+
"""
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# reusable parser types
|
|
29
|
+
PARSERS_TYPE = List[Tuple[Set[str], Set[str], Parser]]
|
|
30
|
+
PARSERS_BY_HEADER_TYPE = Optional[Dict[str, Dict[str, Parser]]]
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
import codecs
|
|
5
|
+
import logging
|
|
6
|
+
from dataclasses import InitVar, dataclass
|
|
7
|
+
from gzip import decompress
|
|
8
|
+
from typing import Any, Generator, List, Mapping, MutableMapping, Optional
|
|
9
|
+
|
|
10
|
+
import orjson
|
|
11
|
+
import requests
|
|
12
|
+
|
|
13
|
+
from airbyte_cdk.sources.declarative.decoders import CompositeRawDecoder, JsonParser
|
|
14
|
+
from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger("airbyte")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class JsonDecoder(Decoder):
|
|
20
|
+
"""
|
|
21
|
+
Decoder strategy that returns the json-encoded content of a response, if any.
|
|
22
|
+
|
|
23
|
+
Usually, we would try to instantiate the equivalent `CompositeRawDecoder(parser=JsonParser(), stream_response=False)` but there were specific historical behaviors related to the JsonDecoder that we didn't know if we could remove like the fallback on {} in case of errors.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, parameters: Mapping[str, Any]):
|
|
27
|
+
self._decoder = CompositeRawDecoder(parser=JsonParser(), stream_response=False)
|
|
28
|
+
|
|
29
|
+
def is_stream_response(self) -> bool:
|
|
30
|
+
return self._decoder.is_stream_response()
|
|
31
|
+
|
|
32
|
+
def decode(
|
|
33
|
+
self, response: requests.Response
|
|
34
|
+
) -> Generator[MutableMapping[str, Any], None, None]:
|
|
35
|
+
"""
|
|
36
|
+
Given the response is an empty string or an emtpy list, the function will return a generator with an empty mapping.
|
|
37
|
+
"""
|
|
38
|
+
has_yielded = False
|
|
39
|
+
try:
|
|
40
|
+
for element in self._decoder.decode(response):
|
|
41
|
+
yield element
|
|
42
|
+
has_yielded = True
|
|
43
|
+
except Exception:
|
|
44
|
+
yield {}
|
|
45
|
+
|
|
46
|
+
if not has_yielded:
|
|
47
|
+
yield {}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class IterableDecoder(Decoder):
|
|
52
|
+
"""
|
|
53
|
+
Decoder strategy that returns the string content of the response, if any.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
parameters: InitVar[Mapping[str, Any]]
|
|
57
|
+
|
|
58
|
+
def is_stream_response(self) -> bool:
|
|
59
|
+
return True
|
|
60
|
+
|
|
61
|
+
def decode(
|
|
62
|
+
self, response: requests.Response
|
|
63
|
+
) -> Generator[MutableMapping[str, Any], None, None]:
|
|
64
|
+
for line in response.iter_lines():
|
|
65
|
+
yield {"record": line.decode()}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from typing import Any, Generator, Mapping
|
|
5
|
+
|
|
6
|
+
import requests
|
|
7
|
+
|
|
8
|
+
from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger("airbyte")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class NoopDecoder(Decoder):
|
|
14
|
+
def is_stream_response(self) -> bool:
|
|
15
|
+
return False
|
|
16
|
+
|
|
17
|
+
def decode( # type: ignore[override] # Signature doesn't match base class
|
|
18
|
+
self,
|
|
19
|
+
response: requests.Response,
|
|
20
|
+
) -> Generator[Mapping[str, Any], None, None]:
|
|
21
|
+
yield from [{}]
|