airbyte-cdk 6.5.3rc2__py3-none-any.whl → 6.5.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/__init__.py +17 -2
- airbyte_cdk/config_observation.py +10 -3
- airbyte_cdk/connector.py +19 -9
- airbyte_cdk/connector_builder/connector_builder_handler.py +28 -8
- airbyte_cdk/connector_builder/main.py +26 -6
- airbyte_cdk/connector_builder/message_grouper.py +95 -25
- airbyte_cdk/destinations/destination.py +47 -14
- airbyte_cdk/destinations/vector_db_based/config.py +36 -14
- airbyte_cdk/destinations/vector_db_based/document_processor.py +49 -11
- airbyte_cdk/destinations/vector_db_based/embedder.py +52 -11
- airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
- airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +15 -4
- airbyte_cdk/entrypoint.py +82 -26
- airbyte_cdk/exception_handler.py +13 -3
- airbyte_cdk/logger.py +10 -2
- airbyte_cdk/models/airbyte_protocol.py +11 -5
- airbyte_cdk/models/airbyte_protocol_serializers.py +9 -3
- airbyte_cdk/models/well_known_types.py +1 -1
- airbyte_cdk/sources/abstract_source.py +63 -17
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +47 -14
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +25 -7
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +27 -6
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +9 -3
- airbyte_cdk/sources/connector_state_manager.py +32 -10
- airbyte_cdk/sources/declarative/async_job/job.py +3 -1
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +68 -14
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +24 -6
- airbyte_cdk/sources/declarative/async_job/repository.py +3 -1
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +27 -7
- airbyte_cdk/sources/declarative/auth/oauth.py +35 -11
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/token.py +25 -8
- airbyte_cdk/sources/declarative/checks/check_stream.py +12 -4
- airbyte_cdk/sources/declarative/checks/connection_checker.py +3 -1
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +11 -3
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +106 -50
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +20 -6
- airbyte_cdk/sources/declarative/declarative_source.py +3 -1
- airbyte_cdk/sources/declarative/declarative_stream.py +27 -6
- airbyte_cdk/sources/declarative/decoders/decoder.py +3 -1
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +3 -1
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +3 -1
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +6 -2
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +6 -2
- airbyte_cdk/sources/declarative/extractors/record_filter.py +24 -7
- airbyte_cdk/sources/declarative/extractors/record_selector.py +10 -3
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +15 -5
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +96 -31
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +22 -8
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +46 -15
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +19 -5
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +3 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +20 -2
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +5 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +10 -3
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +6 -2
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +7 -1
- airbyte_cdk/sources/declarative/interpolation/jinja.py +6 -2
- airbyte_cdk/sources/declarative/interpolation/macros.py +19 -4
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +106 -24
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +7 -2
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +656 -678
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +13 -4
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +9 -2
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +782 -232
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +29 -7
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +25 -7
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +54 -15
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +6 -2
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +17 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +15 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +18 -8
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +16 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +51 -14
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +29 -8
- airbyte_cdk/sources/declarative/requesters/http_requester.py +58 -16
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +49 -14
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +3 -1
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +3 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +17 -5
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +24 -7
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +9 -3
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +3 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +6 -2
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +19 -6
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +3 -1
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +21 -7
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +18 -6
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +27 -8
- airbyte_cdk/sources/declarative/requesters/requester.py +3 -1
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +12 -5
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +105 -24
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +3 -1
- airbyte_cdk/sources/declarative/spec/spec.py +8 -2
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +3 -1
- airbyte_cdk/sources/declarative/transformations/add_fields.py +12 -3
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +6 -2
- airbyte_cdk/sources/declarative/types.py +8 -1
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +3 -1
- airbyte_cdk/sources/embedded/base_integration.py +14 -4
- airbyte_cdk/sources/embedded/catalog.py +16 -4
- airbyte_cdk/sources/embedded/runner.py +19 -3
- airbyte_cdk/sources/embedded/tools.py +3 -1
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +12 -4
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +27 -7
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +12 -6
- airbyte_cdk/sources/file_based/config/csv_format.py +21 -9
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +6 -2
- airbyte_cdk/sources/file_based/config/unstructured_format.py +10 -3
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
- airbyte_cdk/sources/file_based/exceptions.py +13 -15
- airbyte_cdk/sources/file_based/file_based_source.py +82 -24
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +16 -5
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +58 -17
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +89 -26
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +25 -7
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +8 -2
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +20 -6
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +57 -16
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +64 -15
- airbyte_cdk/sources/file_based/schema_helpers.py +33 -10
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +33 -10
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +47 -11
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +13 -22
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +53 -17
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +17 -5
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +26 -9
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +67 -21
- airbyte_cdk/sources/http_logger.py +5 -1
- airbyte_cdk/sources/message/repository.py +18 -4
- airbyte_cdk/sources/source.py +17 -7
- airbyte_cdk/sources/streams/availability_strategy.py +9 -3
- airbyte_cdk/sources/streams/call_rate.py +63 -19
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +31 -7
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +6 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +77 -22
- airbyte_cdk/sources/streams/concurrent/cursor.py +56 -20
- airbyte_cdk/sources/streams/concurrent/default_stream.py +9 -2
- airbyte_cdk/sources/streams/concurrent/helpers.py +6 -2
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +9 -2
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +4 -1
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +10 -2
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +6 -2
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +25 -10
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +32 -16
- airbyte_cdk/sources/streams/core.py +77 -22
- airbyte_cdk/sources/streams/http/availability_strategy.py +3 -1
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +4 -1
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +3 -1
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +16 -5
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +9 -3
- airbyte_cdk/sources/streams/http/exceptions.py +2 -2
- airbyte_cdk/sources/streams/http/http.py +133 -33
- airbyte_cdk/sources/streams/http/http_client.py +91 -29
- airbyte_cdk/sources/streams/http/rate_limiting.py +23 -7
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +19 -6
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +38 -11
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
- airbyte_cdk/sources/types.py +5 -1
- airbyte_cdk/sources/utils/record_helper.py +12 -3
- airbyte_cdk/sources/utils/schema_helpers.py +9 -3
- airbyte_cdk/sources/utils/slice_logger.py +4 -1
- airbyte_cdk/sources/utils/transform.py +24 -9
- airbyte_cdk/sql/exceptions.py +19 -6
- airbyte_cdk/sql/secrets.py +3 -1
- airbyte_cdk/sql/shared/catalog_providers.py +13 -4
- airbyte_cdk/sql/shared/sql_processor.py +44 -14
- airbyte_cdk/test/catalog_builder.py +19 -8
- airbyte_cdk/test/entrypoint_wrapper.py +27 -8
- airbyte_cdk/test/mock_http/mocker.py +41 -11
- airbyte_cdk/test/mock_http/request.py +9 -3
- airbyte_cdk/test/mock_http/response.py +3 -1
- airbyte_cdk/test/mock_http/response_builder.py +29 -7
- airbyte_cdk/test/state_builder.py +10 -2
- airbyte_cdk/test/utils/data.py +6 -2
- airbyte_cdk/test/utils/http_mocking.py +3 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +3 -1
- airbyte_cdk/utils/analytics_message.py +10 -2
- airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
- airbyte_cdk/utils/mapping_helpers.py +3 -1
- airbyte_cdk/utils/message_utils.py +11 -4
- airbyte_cdk/utils/print_buffer.py +6 -1
- airbyte_cdk/utils/schema_inferrer.py +30 -9
- airbyte_cdk/utils/spec_schema_transformations.py +3 -1
- airbyte_cdk/utils/traced_exception.py +35 -9
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/METADATA +7 -6
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/RECORD +198 -198
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/WHEEL +0 -0
@@ -9,7 +9,21 @@ import importlib
|
|
9
9
|
import inspect
|
10
10
|
import re
|
11
11
|
from functools import partial
|
12
|
-
from typing import
|
12
|
+
from typing import (
|
13
|
+
Any,
|
14
|
+
Callable,
|
15
|
+
Dict,
|
16
|
+
List,
|
17
|
+
Mapping,
|
18
|
+
MutableMapping,
|
19
|
+
Optional,
|
20
|
+
Tuple,
|
21
|
+
Type,
|
22
|
+
Union,
|
23
|
+
get_args,
|
24
|
+
get_origin,
|
25
|
+
get_type_hints,
|
26
|
+
)
|
13
27
|
|
14
28
|
from airbyte_cdk.models import FailureType, Level
|
15
29
|
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
@@ -18,9 +32,14 @@ from airbyte_cdk.sources.declarative.async_job.job_tracker import JobTracker
|
|
18
32
|
from airbyte_cdk.sources.declarative.async_job.repository import AsyncJobRepository
|
19
33
|
from airbyte_cdk.sources.declarative.async_job.status import AsyncJobStatus
|
20
34
|
from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator, JwtAuthenticator
|
21
|
-
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import
|
35
|
+
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import (
|
36
|
+
DeclarativeAuthenticator,
|
37
|
+
NoAuth,
|
38
|
+
)
|
22
39
|
from airbyte_cdk.sources.declarative.auth.jwt import JwtAlgorithm
|
23
|
-
from airbyte_cdk.sources.declarative.auth.oauth import
|
40
|
+
from airbyte_cdk.sources.declarative.auth.oauth import (
|
41
|
+
DeclarativeSingleUseRefreshTokenOauth2Authenticator,
|
42
|
+
)
|
24
43
|
from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator
|
25
44
|
from airbyte_cdk.sources.declarative.auth.token import (
|
26
45
|
ApiKeyAuthenticator,
|
@@ -28,7 +47,11 @@ from airbyte_cdk.sources.declarative.auth.token import (
|
|
28
47
|
BearerAuthenticator,
|
29
48
|
LegacySessionTokenAuthenticator,
|
30
49
|
)
|
31
|
-
from airbyte_cdk.sources.declarative.auth.token_provider import
|
50
|
+
from airbyte_cdk.sources.declarative.auth.token_provider import (
|
51
|
+
InterpolatedStringTokenProvider,
|
52
|
+
SessionTokenProvider,
|
53
|
+
TokenProvider,
|
54
|
+
)
|
32
55
|
from airbyte_cdk.sources.declarative.checks import CheckStream
|
33
56
|
from airbyte_cdk.sources.declarative.concurrency_level import ConcurrencyLevel
|
34
57
|
from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime
|
@@ -41,9 +64,18 @@ from airbyte_cdk.sources.declarative.decoders import (
|
|
41
64
|
PaginationDecoderDecorator,
|
42
65
|
XmlDecoder,
|
43
66
|
)
|
44
|
-
from airbyte_cdk.sources.declarative.extractors import
|
45
|
-
|
46
|
-
|
67
|
+
from airbyte_cdk.sources.declarative.extractors import (
|
68
|
+
DpathExtractor,
|
69
|
+
RecordFilter,
|
70
|
+
RecordSelector,
|
71
|
+
ResponseToFileExtractor,
|
72
|
+
)
|
73
|
+
from airbyte_cdk.sources.declarative.extractors.record_filter import (
|
74
|
+
ClientSideIncrementalRecordFilterDecorator,
|
75
|
+
)
|
76
|
+
from airbyte_cdk.sources.declarative.extractors.record_selector import (
|
77
|
+
SCHEMA_TRANSFORMER_TYPE_MAPPING,
|
78
|
+
)
|
47
79
|
from airbyte_cdk.sources.declarative.incremental import (
|
48
80
|
ChildPartitionResumableFullRefreshCursor,
|
49
81
|
CursorFactory,
|
@@ -56,88 +88,216 @@ from airbyte_cdk.sources.declarative.incremental import (
|
|
56
88
|
)
|
57
89
|
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
58
90
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
|
59
|
-
from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_migration import
|
91
|
+
from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_migration import (
|
92
|
+
LegacyToPerPartitionStateMigration,
|
93
|
+
)
|
60
94
|
from airbyte_cdk.sources.declarative.models import CustomStateMigration
|
61
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
62
|
-
|
63
|
-
|
64
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
65
|
-
|
66
|
-
|
67
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
68
|
-
|
69
|
-
|
70
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
71
|
-
|
72
|
-
|
73
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
74
|
-
|
75
|
-
|
76
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
77
|
-
|
78
|
-
|
79
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
80
|
-
|
81
|
-
|
82
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
83
|
-
|
84
|
-
|
85
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
86
|
-
|
87
|
-
|
88
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
89
|
-
|
95
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
96
|
+
AddedFieldDefinition as AddedFieldDefinitionModel,
|
97
|
+
)
|
98
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
99
|
+
AddFields as AddFieldsModel,
|
100
|
+
)
|
101
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
102
|
+
ApiKeyAuthenticator as ApiKeyAuthenticatorModel,
|
103
|
+
)
|
104
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
105
|
+
AsyncJobStatusMap as AsyncJobStatusMapModel,
|
106
|
+
)
|
107
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
108
|
+
AsyncRetriever as AsyncRetrieverModel,
|
109
|
+
)
|
110
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
111
|
+
BasicHttpAuthenticator as BasicHttpAuthenticatorModel,
|
112
|
+
)
|
113
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
114
|
+
BearerAuthenticator as BearerAuthenticatorModel,
|
115
|
+
)
|
116
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
117
|
+
CheckStream as CheckStreamModel,
|
118
|
+
)
|
119
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
120
|
+
CompositeErrorHandler as CompositeErrorHandlerModel,
|
121
|
+
)
|
122
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
123
|
+
ConcurrencyLevel as ConcurrencyLevelModel,
|
124
|
+
)
|
125
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
126
|
+
ConstantBackoffStrategy as ConstantBackoffStrategyModel,
|
127
|
+
)
|
128
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
129
|
+
CursorPagination as CursorPaginationModel,
|
130
|
+
)
|
131
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
132
|
+
CustomAuthenticator as CustomAuthenticatorModel,
|
133
|
+
)
|
134
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
135
|
+
CustomBackoffStrategy as CustomBackoffStrategyModel,
|
136
|
+
)
|
137
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
138
|
+
CustomErrorHandler as CustomErrorHandlerModel,
|
139
|
+
)
|
140
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
141
|
+
CustomIncrementalSync as CustomIncrementalSyncModel,
|
142
|
+
)
|
143
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
144
|
+
CustomPaginationStrategy as CustomPaginationStrategyModel,
|
145
|
+
)
|
146
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
147
|
+
CustomPartitionRouter as CustomPartitionRouterModel,
|
148
|
+
)
|
149
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
150
|
+
CustomRecordExtractor as CustomRecordExtractorModel,
|
151
|
+
)
|
152
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
153
|
+
CustomRecordFilter as CustomRecordFilterModel,
|
154
|
+
)
|
155
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
156
|
+
CustomRequester as CustomRequesterModel,
|
157
|
+
)
|
158
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
159
|
+
CustomRetriever as CustomRetrieverModel,
|
160
|
+
)
|
161
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
162
|
+
CustomSchemaLoader as CustomSchemaLoader,
|
163
|
+
)
|
164
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
165
|
+
CustomTransformation as CustomTransformationModel,
|
166
|
+
)
|
167
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
168
|
+
DatetimeBasedCursor as DatetimeBasedCursorModel,
|
169
|
+
)
|
170
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
171
|
+
DeclarativeStream as DeclarativeStreamModel,
|
172
|
+
)
|
173
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
174
|
+
DefaultErrorHandler as DefaultErrorHandlerModel,
|
175
|
+
)
|
176
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
177
|
+
DefaultPaginator as DefaultPaginatorModel,
|
178
|
+
)
|
179
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
180
|
+
DpathExtractor as DpathExtractorModel,
|
181
|
+
)
|
90
182
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
91
183
|
ExponentialBackoffStrategy as ExponentialBackoffStrategyModel,
|
92
184
|
)
|
93
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
94
|
-
|
95
|
-
|
96
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
97
|
-
|
98
|
-
|
99
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
100
|
-
|
101
|
-
|
102
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
103
|
-
|
185
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
186
|
+
HttpRequester as HttpRequesterModel,
|
187
|
+
)
|
188
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
189
|
+
HttpResponseFilter as HttpResponseFilterModel,
|
190
|
+
)
|
191
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
192
|
+
InlineSchemaLoader as InlineSchemaLoaderModel,
|
193
|
+
)
|
194
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
195
|
+
IterableDecoder as IterableDecoderModel,
|
196
|
+
)
|
197
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
198
|
+
JsonDecoder as JsonDecoderModel,
|
199
|
+
)
|
200
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
201
|
+
JsonFileSchemaLoader as JsonFileSchemaLoaderModel,
|
202
|
+
)
|
203
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
204
|
+
JsonlDecoder as JsonlDecoderModel,
|
205
|
+
)
|
206
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
207
|
+
JwtAuthenticator as JwtAuthenticatorModel,
|
208
|
+
)
|
209
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
210
|
+
JwtHeaders as JwtHeadersModel,
|
211
|
+
)
|
212
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
213
|
+
JwtPayload as JwtPayloadModel,
|
214
|
+
)
|
215
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
216
|
+
KeysToLower as KeysToLowerModel,
|
217
|
+
)
|
104
218
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
105
219
|
LegacySessionTokenAuthenticator as LegacySessionTokenAuthenticatorModel,
|
106
220
|
)
|
107
221
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
108
222
|
LegacyToPerPartitionStateMigration as LegacyToPerPartitionStateMigrationModel,
|
109
223
|
)
|
110
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
111
|
-
|
112
|
-
|
113
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
114
|
-
|
115
|
-
|
116
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
117
|
-
|
118
|
-
|
119
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
120
|
-
|
121
|
-
|
122
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
123
|
-
|
124
|
-
|
125
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
224
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
225
|
+
ListPartitionRouter as ListPartitionRouterModel,
|
226
|
+
)
|
227
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
228
|
+
MinMaxDatetime as MinMaxDatetimeModel,
|
229
|
+
)
|
230
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
231
|
+
NoAuth as NoAuthModel,
|
232
|
+
)
|
233
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
234
|
+
NoPagination as NoPaginationModel,
|
235
|
+
)
|
236
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
237
|
+
OAuthAuthenticator as OAuthAuthenticatorModel,
|
238
|
+
)
|
239
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
240
|
+
OffsetIncrement as OffsetIncrementModel,
|
241
|
+
)
|
242
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
243
|
+
PageIncrement as PageIncrementModel,
|
244
|
+
)
|
245
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
246
|
+
ParentStreamConfig as ParentStreamConfigModel,
|
247
|
+
)
|
248
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
249
|
+
RecordFilter as RecordFilterModel,
|
250
|
+
)
|
251
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
252
|
+
RecordSelector as RecordSelectorModel,
|
253
|
+
)
|
254
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
255
|
+
RemoveFields as RemoveFieldsModel,
|
256
|
+
)
|
257
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
258
|
+
RequestOption as RequestOptionModel,
|
259
|
+
)
|
260
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
261
|
+
RequestPath as RequestPathModel,
|
262
|
+
)
|
263
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
264
|
+
SelectiveAuthenticator as SelectiveAuthenticatorModel,
|
265
|
+
)
|
266
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
267
|
+
SessionTokenAuthenticator as SessionTokenAuthenticatorModel,
|
268
|
+
)
|
269
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
270
|
+
SimpleRetriever as SimpleRetrieverModel,
|
271
|
+
)
|
126
272
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel
|
127
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
273
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
274
|
+
SubstreamPartitionRouter as SubstreamPartitionRouterModel,
|
275
|
+
)
|
128
276
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType
|
129
|
-
from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
130
|
-
|
131
|
-
|
277
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
278
|
+
WaitTimeFromHeader as WaitTimeFromHeaderModel,
|
279
|
+
)
|
280
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
281
|
+
WaitUntilTimeFromHeader as WaitUntilTimeFromHeaderModel,
|
282
|
+
)
|
283
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
284
|
+
XmlDecoder as XmlDecoderModel,
|
285
|
+
)
|
132
286
|
from airbyte_cdk.sources.declarative.partition_routers import (
|
133
287
|
CartesianProductStreamSlicer,
|
134
288
|
ListPartitionRouter,
|
135
289
|
SinglePartitionRouter,
|
136
290
|
SubstreamPartitionRouter,
|
137
291
|
)
|
138
|
-
from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import
|
292
|
+
from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import (
|
293
|
+
ParentStreamConfig,
|
294
|
+
)
|
139
295
|
from airbyte_cdk.sources.declarative.requesters import HttpRequester, RequestOption
|
140
|
-
from airbyte_cdk.sources.declarative.requesters.error_handlers import
|
296
|
+
from airbyte_cdk.sources.declarative.requesters.error_handlers import (
|
297
|
+
CompositeErrorHandler,
|
298
|
+
DefaultErrorHandler,
|
299
|
+
HttpResponseFilter,
|
300
|
+
)
|
141
301
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import (
|
142
302
|
ConstantBackoffStrategy,
|
143
303
|
ExponentialBackoffStrategy,
|
@@ -145,7 +305,11 @@ from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategie
|
|
145
305
|
WaitUntilTimeFromHeaderBackoffStrategy,
|
146
306
|
)
|
147
307
|
from airbyte_cdk.sources.declarative.requesters.http_job_repository import AsyncHttpJobRepository
|
148
|
-
from airbyte_cdk.sources.declarative.requesters.paginators import
|
308
|
+
from airbyte_cdk.sources.declarative.requesters.paginators import (
|
309
|
+
DefaultPaginator,
|
310
|
+
NoPagination,
|
311
|
+
PaginatorTestReadDecorator,
|
312
|
+
)
|
149
313
|
from airbyte_cdk.sources.declarative.requesters.paginators.strategies import (
|
150
314
|
CursorPaginationStrategy,
|
151
315
|
CursorStopCondition,
|
@@ -162,14 +326,32 @@ from airbyte_cdk.sources.declarative.requesters.request_options import (
|
|
162
326
|
)
|
163
327
|
from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
|
164
328
|
from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
|
165
|
-
from airbyte_cdk.sources.declarative.retrievers import
|
166
|
-
|
329
|
+
from airbyte_cdk.sources.declarative.retrievers import (
|
330
|
+
AsyncRetriever,
|
331
|
+
SimpleRetriever,
|
332
|
+
SimpleRetrieverTestReadDecorator,
|
333
|
+
)
|
334
|
+
from airbyte_cdk.sources.declarative.schema import (
|
335
|
+
DefaultSchemaLoader,
|
336
|
+
InlineSchemaLoader,
|
337
|
+
JsonFileSchemaLoader,
|
338
|
+
)
|
167
339
|
from airbyte_cdk.sources.declarative.spec import Spec
|
168
340
|
from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer
|
169
|
-
from airbyte_cdk.sources.declarative.transformations import
|
341
|
+
from airbyte_cdk.sources.declarative.transformations import (
|
342
|
+
AddFields,
|
343
|
+
RecordTransformation,
|
344
|
+
RemoveFields,
|
345
|
+
)
|
170
346
|
from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
|
171
|
-
from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation import
|
172
|
-
|
347
|
+
from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformation import (
|
348
|
+
KeysToLowerTransformation,
|
349
|
+
)
|
350
|
+
from airbyte_cdk.sources.message import (
|
351
|
+
InMemoryMessageRepository,
|
352
|
+
LogAppenderMessageRepositoryDecorator,
|
353
|
+
MessageRepository,
|
354
|
+
)
|
173
355
|
from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, CursorField
|
174
356
|
from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import (
|
175
357
|
CustomFormatConcurrentStreamStateConverter,
|
@@ -185,7 +367,6 @@ ComponentDefinition = Mapping[str, Any]
|
|
185
367
|
|
186
368
|
|
187
369
|
class ModelToComponentFactory:
|
188
|
-
|
189
370
|
EPOCH_DATETIME_FORMAT = "%s"
|
190
371
|
|
191
372
|
def __init__(
|
@@ -277,7 +458,11 @@ class ModelToComponentFactory:
|
|
277
458
|
self.TYPE_NAME_TO_MODEL = {cls.__name__: cls for cls in self.PYDANTIC_MODEL_TO_CONSTRUCTOR}
|
278
459
|
|
279
460
|
def create_component(
|
280
|
-
self,
|
461
|
+
self,
|
462
|
+
model_type: Type[BaseModel],
|
463
|
+
component_definition: ComponentDefinition,
|
464
|
+
config: Config,
|
465
|
+
**kwargs: Any,
|
281
466
|
) -> Any:
|
282
467
|
"""
|
283
468
|
Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and
|
@@ -292,26 +477,38 @@ class ModelToComponentFactory:
|
|
292
477
|
|
293
478
|
component_type = component_definition.get("type")
|
294
479
|
if component_definition.get("type") != model_type.__name__:
|
295
|
-
raise ValueError(
|
480
|
+
raise ValueError(
|
481
|
+
f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead"
|
482
|
+
)
|
296
483
|
|
297
484
|
declarative_component_model = model_type.parse_obj(component_definition)
|
298
485
|
|
299
486
|
if not isinstance(declarative_component_model, model_type):
|
300
|
-
raise ValueError(
|
487
|
+
raise ValueError(
|
488
|
+
f"Expected {model_type.__name__} component, but received {declarative_component_model.__class__.__name__}"
|
489
|
+
)
|
301
490
|
|
302
|
-
return self._create_component_from_model(
|
491
|
+
return self._create_component_from_model(
|
492
|
+
model=declarative_component_model, config=config, **kwargs
|
493
|
+
)
|
303
494
|
|
304
495
|
def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs: Any) -> Any:
|
305
496
|
if model.__class__ not in self.PYDANTIC_MODEL_TO_CONSTRUCTOR:
|
306
|
-
raise ValueError(
|
497
|
+
raise ValueError(
|
498
|
+
f"{model.__class__} with attributes {model} is not a valid component type"
|
499
|
+
)
|
307
500
|
component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__)
|
308
501
|
if not component_constructor:
|
309
502
|
raise ValueError(f"Could not find constructor for {model.__class__}")
|
310
503
|
return component_constructor(model=model, config=config, **kwargs)
|
311
504
|
|
312
505
|
@staticmethod
|
313
|
-
def create_added_field_definition(
|
314
|
-
|
506
|
+
def create_added_field_definition(
|
507
|
+
model: AddedFieldDefinitionModel, config: Config, **kwargs: Any
|
508
|
+
) -> AddedFieldDefinition:
|
509
|
+
interpolated_value = InterpolatedString.create(
|
510
|
+
model.value, parameters=model.parameters or {}
|
511
|
+
)
|
315
512
|
return AddedFieldDefinition(
|
316
513
|
path=model.path,
|
317
514
|
value=interpolated_value,
|
@@ -323,14 +520,18 @@ class ModelToComponentFactory:
|
|
323
520
|
added_field_definitions = [
|
324
521
|
self._create_component_from_model(
|
325
522
|
model=added_field_definition_model,
|
326
|
-
value_type=ModelToComponentFactory._json_schema_type_name_to_type(
|
523
|
+
value_type=ModelToComponentFactory._json_schema_type_name_to_type(
|
524
|
+
added_field_definition_model.value_type
|
525
|
+
),
|
327
526
|
config=config,
|
328
527
|
)
|
329
528
|
for added_field_definition_model in model.fields
|
330
529
|
]
|
331
530
|
return AddFields(fields=added_field_definitions, parameters=model.parameters or {})
|
332
531
|
|
333
|
-
def create_keys_to_lower_transformation(
|
532
|
+
def create_keys_to_lower_transformation(
|
533
|
+
self, model: KeysToLowerModel, config: Config, **kwargs: Any
|
534
|
+
) -> KeysToLowerTransformation:
|
334
535
|
return KeysToLowerTransformation()
|
335
536
|
|
336
537
|
@staticmethod
|
@@ -347,16 +548,25 @@ class ModelToComponentFactory:
|
|
347
548
|
|
348
549
|
@staticmethod
|
349
550
|
def create_api_key_authenticator(
|
350
|
-
model: ApiKeyAuthenticatorModel,
|
551
|
+
model: ApiKeyAuthenticatorModel,
|
552
|
+
config: Config,
|
553
|
+
token_provider: Optional[TokenProvider] = None,
|
554
|
+
**kwargs: Any,
|
351
555
|
) -> ApiKeyAuthenticator:
|
352
556
|
if model.inject_into is None and model.header is None:
|
353
|
-
raise ValueError(
|
557
|
+
raise ValueError(
|
558
|
+
"Expected either inject_into or header to be set for ApiKeyAuthenticator"
|
559
|
+
)
|
354
560
|
|
355
561
|
if model.inject_into is not None and model.header is not None:
|
356
|
-
raise ValueError(
|
562
|
+
raise ValueError(
|
563
|
+
"inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option"
|
564
|
+
)
|
357
565
|
|
358
566
|
if token_provider is not None and model.api_token != "":
|
359
|
-
raise ValueError(
|
567
|
+
raise ValueError(
|
568
|
+
"If token_provider is set, api_token is ignored and has to be set to empty string."
|
569
|
+
)
|
360
570
|
|
361
571
|
request_option = (
|
362
572
|
RequestOption(
|
@@ -375,7 +585,11 @@ class ModelToComponentFactory:
|
|
375
585
|
token_provider=(
|
376
586
|
token_provider
|
377
587
|
if token_provider is not None
|
378
|
-
else InterpolatedStringTokenProvider(
|
588
|
+
else InterpolatedStringTokenProvider(
|
589
|
+
api_token=model.api_token or "",
|
590
|
+
config=config,
|
591
|
+
parameters=model.parameters or {},
|
592
|
+
)
|
379
593
|
),
|
380
594
|
request_option=request_option,
|
381
595
|
config=config,
|
@@ -394,26 +608,44 @@ class ModelToComponentFactory:
|
|
394
608
|
f"LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever. Got {type(retriever)}"
|
395
609
|
)
|
396
610
|
partition_router = retriever.partition_router
|
397
|
-
if not isinstance(
|
611
|
+
if not isinstance(
|
612
|
+
partition_router, (SubstreamPartitionRouterModel, CustomPartitionRouterModel)
|
613
|
+
):
|
398
614
|
raise ValueError(
|
399
615
|
f"LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got {type(partition_router)}"
|
400
616
|
)
|
401
617
|
if not hasattr(partition_router, "parent_stream_configs"):
|
402
|
-
raise ValueError(
|
618
|
+
raise ValueError(
|
619
|
+
"LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration."
|
620
|
+
)
|
403
621
|
|
404
|
-
return LegacyToPerPartitionStateMigration(
|
622
|
+
return LegacyToPerPartitionStateMigration(
|
623
|
+
declarative_stream.retriever.partition_router,
|
624
|
+
declarative_stream.incremental_sync,
|
625
|
+
config,
|
626
|
+
declarative_stream.parameters,
|
627
|
+
) # type: ignore # The retriever type was already checked
|
405
628
|
|
406
629
|
def create_session_token_authenticator(
|
407
630
|
self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any
|
408
631
|
) -> Union[ApiKeyAuthenticator, BearerAuthenticator]:
|
409
|
-
decoder =
|
632
|
+
decoder = (
|
633
|
+
self._create_component_from_model(model=model.decoder, config=config)
|
634
|
+
if model.decoder
|
635
|
+
else JsonDecoder(parameters={})
|
636
|
+
)
|
410
637
|
login_requester = self._create_component_from_model(
|
411
|
-
model=model.login_requester,
|
638
|
+
model=model.login_requester,
|
639
|
+
config=config,
|
640
|
+
name=f"{name}_login_requester",
|
641
|
+
decoder=decoder,
|
412
642
|
)
|
413
643
|
token_provider = SessionTokenProvider(
|
414
644
|
login_requester=login_requester,
|
415
645
|
session_token_path=model.session_token_path,
|
416
|
-
expiration_duration=parse_duration(model.expiration_duration)
|
646
|
+
expiration_duration=parse_duration(model.expiration_duration)
|
647
|
+
if model.expiration_duration
|
648
|
+
else None,
|
417
649
|
parameters=model.parameters or {},
|
418
650
|
message_repository=self._message_repository,
|
419
651
|
decoder=decoder,
|
@@ -426,28 +658,46 @@ class ModelToComponentFactory:
|
|
426
658
|
)
|
427
659
|
else:
|
428
660
|
return ModelToComponentFactory.create_api_key_authenticator(
|
429
|
-
ApiKeyAuthenticatorModel(
|
661
|
+
ApiKeyAuthenticatorModel(
|
662
|
+
type="ApiKeyAuthenticator",
|
663
|
+
api_token="",
|
664
|
+
inject_into=model.request_authentication.inject_into,
|
665
|
+
), # type: ignore # $parameters and headers default to None
|
430
666
|
config=config,
|
431
667
|
token_provider=token_provider,
|
432
668
|
)
|
433
669
|
|
434
670
|
@staticmethod
|
435
|
-
def create_basic_http_authenticator(
|
671
|
+
def create_basic_http_authenticator(
|
672
|
+
model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any
|
673
|
+
) -> BasicHttpAuthenticator:
|
436
674
|
return BasicHttpAuthenticator(
|
437
|
-
password=model.password or "",
|
675
|
+
password=model.password or "",
|
676
|
+
username=model.username,
|
677
|
+
config=config,
|
678
|
+
parameters=model.parameters or {},
|
438
679
|
)
|
439
680
|
|
440
681
|
@staticmethod
|
441
682
|
def create_bearer_authenticator(
|
442
|
-
model: BearerAuthenticatorModel,
|
683
|
+
model: BearerAuthenticatorModel,
|
684
|
+
config: Config,
|
685
|
+
token_provider: Optional[TokenProvider] = None,
|
686
|
+
**kwargs: Any,
|
443
687
|
) -> BearerAuthenticator:
|
444
688
|
if token_provider is not None and model.api_token != "":
|
445
|
-
raise ValueError(
|
689
|
+
raise ValueError(
|
690
|
+
"If token_provider is set, api_token is ignored and has to be set to empty string."
|
691
|
+
)
|
446
692
|
return BearerAuthenticator(
|
447
693
|
token_provider=(
|
448
694
|
token_provider
|
449
695
|
if token_provider is not None
|
450
|
-
else InterpolatedStringTokenProvider(
|
696
|
+
else InterpolatedStringTokenProvider(
|
697
|
+
api_token=model.api_token or "",
|
698
|
+
config=config,
|
699
|
+
parameters=model.parameters or {},
|
700
|
+
)
|
451
701
|
),
|
452
702
|
config=config,
|
453
703
|
parameters=model.parameters or {},
|
@@ -457,14 +707,21 @@ class ModelToComponentFactory:
|
|
457
707
|
def create_check_stream(model: CheckStreamModel, config: Config, **kwargs: Any) -> CheckStream:
|
458
708
|
return CheckStream(stream_names=model.stream_names, parameters={})
|
459
709
|
|
460
|
-
def create_composite_error_handler(
|
710
|
+
def create_composite_error_handler(
|
711
|
+
self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any
|
712
|
+
) -> CompositeErrorHandler:
|
461
713
|
error_handlers = [
|
462
|
-
self._create_component_from_model(model=error_handler_model, config=config)
|
714
|
+
self._create_component_from_model(model=error_handler_model, config=config)
|
715
|
+
for error_handler_model in model.error_handlers
|
463
716
|
]
|
464
|
-
return CompositeErrorHandler(
|
717
|
+
return CompositeErrorHandler(
|
718
|
+
error_handlers=error_handlers, parameters=model.parameters or {}
|
719
|
+
)
|
465
720
|
|
466
721
|
@staticmethod
|
467
|
-
def create_concurrency_level(
|
722
|
+
def create_concurrency_level(
|
723
|
+
model: ConcurrencyLevelModel, config: Config, **kwargs: Any
|
724
|
+
) -> ConcurrencyLevel:
|
468
725
|
return ConcurrencyLevel(
|
469
726
|
default_concurrency=model.default_concurrency,
|
470
727
|
max_concurrency=model.max_concurrency,
|
@@ -483,26 +740,32 @@ class ModelToComponentFactory:
|
|
483
740
|
stream_state: MutableMapping[str, Any],
|
484
741
|
**kwargs: Any,
|
485
742
|
) -> Tuple[ConcurrentCursor, DateTimeStreamStateConverter]:
|
486
|
-
|
487
743
|
component_type = component_definition.get("type")
|
488
744
|
if component_definition.get("type") != model_type.__name__:
|
489
|
-
raise ValueError(
|
745
|
+
raise ValueError(
|
746
|
+
f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead"
|
747
|
+
)
|
490
748
|
|
491
749
|
datetime_based_cursor_model = model_type.parse_obj(component_definition)
|
492
750
|
|
493
751
|
if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel):
|
494
|
-
raise ValueError(
|
752
|
+
raise ValueError(
|
753
|
+
f"Expected {model_type.__name__} component, but received {datetime_based_cursor_model.__class__.__name__}"
|
754
|
+
)
|
495
755
|
|
496
756
|
interpolated_cursor_field = InterpolatedString.create(
|
497
|
-
datetime_based_cursor_model.cursor_field,
|
757
|
+
datetime_based_cursor_model.cursor_field,
|
758
|
+
parameters=datetime_based_cursor_model.parameters or {},
|
498
759
|
)
|
499
760
|
cursor_field = CursorField(interpolated_cursor_field.eval(config=config))
|
500
761
|
|
501
762
|
interpolated_partition_field_start = InterpolatedString.create(
|
502
|
-
datetime_based_cursor_model.partition_field_start or "start_time",
|
763
|
+
datetime_based_cursor_model.partition_field_start or "start_time",
|
764
|
+
parameters=datetime_based_cursor_model.parameters or {},
|
503
765
|
)
|
504
766
|
interpolated_partition_field_end = InterpolatedString.create(
|
505
|
-
datetime_based_cursor_model.partition_field_end or "end_time",
|
767
|
+
datetime_based_cursor_model.partition_field_end or "end_time",
|
768
|
+
parameters=datetime_based_cursor_model.parameters or {},
|
506
769
|
)
|
507
770
|
|
508
771
|
slice_boundary_fields = (
|
@@ -513,12 +776,17 @@ class ModelToComponentFactory:
|
|
513
776
|
datetime_format = datetime_based_cursor_model.datetime_format
|
514
777
|
|
515
778
|
cursor_granularity = (
|
516
|
-
parse_duration(datetime_based_cursor_model.cursor_granularity)
|
779
|
+
parse_duration(datetime_based_cursor_model.cursor_granularity)
|
780
|
+
if datetime_based_cursor_model.cursor_granularity
|
781
|
+
else None
|
517
782
|
)
|
518
783
|
|
519
784
|
lookback_window = None
|
520
785
|
interpolated_lookback_window = (
|
521
|
-
InterpolatedString.create(
|
786
|
+
InterpolatedString.create(
|
787
|
+
datetime_based_cursor_model.lookback_window,
|
788
|
+
parameters=datetime_based_cursor_model.parameters or {},
|
789
|
+
)
|
522
790
|
if datetime_based_cursor_model.lookback_window
|
523
791
|
else None
|
524
792
|
)
|
@@ -538,21 +806,30 @@ class ModelToComponentFactory:
|
|
538
806
|
|
539
807
|
start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime]
|
540
808
|
if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel):
|
541
|
-
start_date_runtime_value = self.create_min_max_datetime(
|
809
|
+
start_date_runtime_value = self.create_min_max_datetime(
|
810
|
+
model=datetime_based_cursor_model.start_datetime, config=config
|
811
|
+
)
|
542
812
|
else:
|
543
813
|
start_date_runtime_value = datetime_based_cursor_model.start_datetime
|
544
814
|
|
545
815
|
end_date_runtime_value: Optional[Union[InterpolatedString, str, MinMaxDatetime]]
|
546
816
|
if isinstance(datetime_based_cursor_model.end_datetime, MinMaxDatetimeModel):
|
547
|
-
end_date_runtime_value = self.create_min_max_datetime(
|
817
|
+
end_date_runtime_value = self.create_min_max_datetime(
|
818
|
+
model=datetime_based_cursor_model.end_datetime, config=config
|
819
|
+
)
|
548
820
|
else:
|
549
821
|
end_date_runtime_value = datetime_based_cursor_model.end_datetime
|
550
822
|
|
551
823
|
interpolated_start_date = MinMaxDatetime.create(
|
552
|
-
interpolated_string_or_min_max_datetime=start_date_runtime_value,
|
824
|
+
interpolated_string_or_min_max_datetime=start_date_runtime_value,
|
825
|
+
parameters=datetime_based_cursor_model.parameters,
|
553
826
|
)
|
554
827
|
interpolated_end_date = (
|
555
|
-
None
|
828
|
+
None
|
829
|
+
if not end_date_runtime_value
|
830
|
+
else MinMaxDatetime.create(
|
831
|
+
end_date_runtime_value, datetime_based_cursor_model.parameters
|
832
|
+
)
|
556
833
|
)
|
557
834
|
|
558
835
|
# If datetime format is not specified then start/end datetime should inherit it from the stream slicer
|
@@ -563,10 +840,14 @@ class ModelToComponentFactory:
|
|
563
840
|
|
564
841
|
start_date = interpolated_start_date.get_datetime(config=config)
|
565
842
|
end_date_provider = (
|
566
|
-
partial(interpolated_end_date.get_datetime, config)
|
843
|
+
partial(interpolated_end_date.get_datetime, config)
|
844
|
+
if interpolated_end_date
|
845
|
+
else connector_state_converter.get_end_provider()
|
567
846
|
)
|
568
847
|
|
569
|
-
if (
|
848
|
+
if (
|
849
|
+
datetime_based_cursor_model.step and not datetime_based_cursor_model.cursor_granularity
|
850
|
+
) or (
|
570
851
|
not datetime_based_cursor_model.step and datetime_based_cursor_model.cursor_granularity
|
571
852
|
):
|
572
853
|
raise ValueError(
|
@@ -577,7 +858,10 @@ class ModelToComponentFactory:
|
|
577
858
|
# When step is not defined, default to a step size from the starting date to the present moment
|
578
859
|
step_length = datetime.timedelta.max
|
579
860
|
interpolated_step = (
|
580
|
-
InterpolatedString.create(
|
861
|
+
InterpolatedString.create(
|
862
|
+
datetime_based_cursor_model.step,
|
863
|
+
parameters=datetime_based_cursor_model.parameters or {},
|
864
|
+
)
|
581
865
|
if datetime_based_cursor_model.step
|
582
866
|
else None
|
583
867
|
)
|
@@ -606,7 +890,9 @@ class ModelToComponentFactory:
|
|
606
890
|
)
|
607
891
|
|
608
892
|
@staticmethod
|
609
|
-
def create_constant_backoff_strategy(
|
893
|
+
def create_constant_backoff_strategy(
|
894
|
+
model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any
|
895
|
+
) -> ConstantBackoffStrategy:
|
610
896
|
return ConstantBackoffStrategy(
|
611
897
|
backoff_time_in_seconds=model.backoff_time_in_seconds,
|
612
898
|
config=config,
|
@@ -624,7 +910,9 @@ class ModelToComponentFactory:
|
|
624
910
|
decoder_to_use = decoder
|
625
911
|
else:
|
626
912
|
if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
|
627
|
-
raise ValueError(
|
913
|
+
raise ValueError(
|
914
|
+
f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
|
915
|
+
)
|
628
916
|
decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
|
629
917
|
|
630
918
|
return CursorPaginationStrategy(
|
@@ -660,18 +948,28 @@ class ModelToComponentFactory:
|
|
660
948
|
# the custom component and this code performs a second parse to convert the sub-fields first into models, then declarative components
|
661
949
|
for model_field, model_value in model_args.items():
|
662
950
|
# If a custom component field doesn't have a type set, we try to use the type hints to infer the type
|
663
|
-
if
|
664
|
-
|
951
|
+
if (
|
952
|
+
isinstance(model_value, dict)
|
953
|
+
and "type" not in model_value
|
954
|
+
and model_field in component_fields
|
955
|
+
):
|
956
|
+
derived_type = self._derive_component_type_from_type_hints(
|
957
|
+
component_fields.get(model_field)
|
958
|
+
)
|
665
959
|
if derived_type:
|
666
960
|
model_value["type"] = derived_type
|
667
961
|
|
668
962
|
if self._is_component(model_value):
|
669
|
-
model_args[model_field] = self._create_nested_component(
|
963
|
+
model_args[model_field] = self._create_nested_component(
|
964
|
+
model, model_field, model_value, config
|
965
|
+
)
|
670
966
|
elif isinstance(model_value, list):
|
671
967
|
vals = []
|
672
968
|
for v in model_value:
|
673
969
|
if isinstance(v, dict) and "type" not in v and model_field in component_fields:
|
674
|
-
derived_type = self._derive_component_type_from_type_hints(
|
970
|
+
derived_type = self._derive_component_type_from_type_hints(
|
971
|
+
component_fields.get(model_field)
|
972
|
+
)
|
675
973
|
if derived_type:
|
676
974
|
v["type"] = derived_type
|
677
975
|
if self._is_component(v):
|
@@ -680,7 +978,11 @@ class ModelToComponentFactory:
|
|
680
978
|
vals.append(v)
|
681
979
|
model_args[model_field] = vals
|
682
980
|
|
683
|
-
kwargs = {
|
981
|
+
kwargs = {
|
982
|
+
class_field: model_args[class_field]
|
983
|
+
for class_field in component_fields.keys()
|
984
|
+
if class_field in model_args
|
985
|
+
}
|
684
986
|
return custom_component_class(**kwargs)
|
685
987
|
|
686
988
|
@staticmethod
|
@@ -724,7 +1026,9 @@ class ModelToComponentFactory:
|
|
724
1026
|
else:
|
725
1027
|
return []
|
726
1028
|
|
727
|
-
def _create_nested_component(
|
1029
|
+
def _create_nested_component(
|
1030
|
+
self, model: Any, model_field: str, model_value: Any, config: Config
|
1031
|
+
) -> Any:
|
728
1032
|
type_name = model_value.get("type", None)
|
729
1033
|
if not type_name:
|
730
1034
|
# If no type is specified, we can assume this is a dictionary object which can be returned instead of a subcomponent
|
@@ -743,16 +1047,29 @@ class ModelToComponentFactory:
|
|
743
1047
|
model_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(parsed_model.__class__)
|
744
1048
|
constructor_kwargs = inspect.getfullargspec(model_constructor).kwonlyargs
|
745
1049
|
model_parameters = model_value.get("$parameters", {})
|
746
|
-
matching_parameters = {
|
747
|
-
|
1050
|
+
matching_parameters = {
|
1051
|
+
kwarg: model_parameters[kwarg]
|
1052
|
+
for kwarg in constructor_kwargs
|
1053
|
+
if kwarg in model_parameters
|
1054
|
+
}
|
1055
|
+
return self._create_component_from_model(
|
1056
|
+
model=parsed_model, config=config, **matching_parameters
|
1057
|
+
)
|
748
1058
|
except TypeError as error:
|
749
1059
|
missing_parameters = self._extract_missing_parameters(error)
|
750
1060
|
if missing_parameters:
|
751
1061
|
raise ValueError(
|
752
1062
|
f"Error creating component '{type_name}' with parent custom component {model.class_name}: Please provide "
|
753
|
-
+ ", ".join(
|
1063
|
+
+ ", ".join(
|
1064
|
+
(
|
1065
|
+
f"{type_name}.$parameters.{parameter}"
|
1066
|
+
for parameter in missing_parameters
|
1067
|
+
)
|
1068
|
+
)
|
754
1069
|
)
|
755
|
-
raise TypeError(
|
1070
|
+
raise TypeError(
|
1071
|
+
f"Error creating component '{type_name}' with parent custom component {model.class_name}: {error}"
|
1072
|
+
)
|
756
1073
|
else:
|
757
1074
|
raise ValueError(
|
758
1075
|
f"Error creating custom component {model.class_name}. Subcomponent creation has not been implemented for '{type_name}'"
|
@@ -762,18 +1079,26 @@ class ModelToComponentFactory:
|
|
762
1079
|
def _is_component(model_value: Any) -> bool:
|
763
1080
|
return isinstance(model_value, dict) and model_value.get("type") is not None
|
764
1081
|
|
765
|
-
def create_datetime_based_cursor(
|
1082
|
+
def create_datetime_based_cursor(
|
1083
|
+
self, model: DatetimeBasedCursorModel, config: Config, **kwargs: Any
|
1084
|
+
) -> DatetimeBasedCursor:
|
766
1085
|
start_datetime: Union[str, MinMaxDatetime] = (
|
767
|
-
model.start_datetime
|
1086
|
+
model.start_datetime
|
1087
|
+
if isinstance(model.start_datetime, str)
|
1088
|
+
else self.create_min_max_datetime(model.start_datetime, config)
|
768
1089
|
)
|
769
1090
|
end_datetime: Union[str, MinMaxDatetime, None] = None
|
770
1091
|
if model.is_data_feed and model.end_datetime:
|
771
1092
|
raise ValueError("Data feed does not support end_datetime")
|
772
1093
|
if model.is_data_feed and model.is_client_side_incremental:
|
773
|
-
raise ValueError(
|
1094
|
+
raise ValueError(
|
1095
|
+
"`Client side incremental` cannot be applied with `data feed`. Choose only 1 from them."
|
1096
|
+
)
|
774
1097
|
if model.end_datetime:
|
775
1098
|
end_datetime = (
|
776
|
-
model.end_datetime
|
1099
|
+
model.end_datetime
|
1100
|
+
if isinstance(model.end_datetime, str)
|
1101
|
+
else self.create_min_max_datetime(model.end_datetime, config)
|
777
1102
|
)
|
778
1103
|
|
779
1104
|
end_time_option = (
|
@@ -797,7 +1122,9 @@ class ModelToComponentFactory:
|
|
797
1122
|
|
798
1123
|
return DatetimeBasedCursor(
|
799
1124
|
cursor_field=model.cursor_field,
|
800
|
-
cursor_datetime_formats=model.cursor_datetime_formats
|
1125
|
+
cursor_datetime_formats=model.cursor_datetime_formats
|
1126
|
+
if model.cursor_datetime_formats
|
1127
|
+
else [],
|
801
1128
|
cursor_granularity=model.cursor_granularity,
|
802
1129
|
datetime_format=model.datetime_format,
|
803
1130
|
end_datetime=end_datetime,
|
@@ -814,7 +1141,9 @@ class ModelToComponentFactory:
|
|
814
1141
|
parameters=model.parameters or {},
|
815
1142
|
)
|
816
1143
|
|
817
|
-
def create_declarative_stream(
|
1144
|
+
def create_declarative_stream(
|
1145
|
+
self, model: DeclarativeStreamModel, config: Config, **kwargs: Any
|
1146
|
+
) -> DeclarativeStream:
|
818
1147
|
# When constructing a declarative stream, we assemble the incremental_sync component and retriever's partition_router field
|
819
1148
|
# components if they exist into a single CartesianProductStreamSlicer. This is then passed back as an argument when constructing the
|
820
1149
|
# Retriever. This is done in the declarative stream not the retriever to support custom retrievers. The custom create methods in
|
@@ -823,7 +1152,9 @@ class ModelToComponentFactory:
|
|
823
1152
|
|
824
1153
|
primary_key = model.primary_key.__root__ if model.primary_key else None
|
825
1154
|
stop_condition_on_cursor = (
|
826
|
-
model.incremental_sync
|
1155
|
+
model.incremental_sync
|
1156
|
+
and hasattr(model.incremental_sync, "is_data_feed")
|
1157
|
+
and model.incremental_sync.is_data_feed
|
827
1158
|
)
|
828
1159
|
client_side_incremental_sync = None
|
829
1160
|
if (
|
@@ -831,13 +1162,25 @@ class ModelToComponentFactory:
|
|
831
1162
|
and hasattr(model.incremental_sync, "is_client_side_incremental")
|
832
1163
|
and model.incremental_sync.is_client_side_incremental
|
833
1164
|
):
|
834
|
-
supported_slicers = (
|
1165
|
+
supported_slicers = (
|
1166
|
+
DatetimeBasedCursor,
|
1167
|
+
GlobalSubstreamCursor,
|
1168
|
+
PerPartitionWithGlobalCursor,
|
1169
|
+
)
|
835
1170
|
if combined_slicers and not isinstance(combined_slicers, supported_slicers):
|
836
|
-
raise ValueError(
|
1171
|
+
raise ValueError(
|
1172
|
+
"Unsupported Slicer is used. PerPartitionWithGlobalCursor should be used here instead"
|
1173
|
+
)
|
837
1174
|
client_side_incremental_sync = {
|
838
|
-
"date_time_based_cursor": self._create_component_from_model(
|
1175
|
+
"date_time_based_cursor": self._create_component_from_model(
|
1176
|
+
model=model.incremental_sync, config=config
|
1177
|
+
),
|
839
1178
|
"substream_cursor": (
|
840
|
-
combined_slicers
|
1179
|
+
combined_slicers
|
1180
|
+
if isinstance(
|
1181
|
+
combined_slicers, (PerPartitionWithGlobalCursor, GlobalSubstreamCursor)
|
1182
|
+
)
|
1183
|
+
else None
|
841
1184
|
),
|
842
1185
|
}
|
843
1186
|
|
@@ -877,7 +1220,9 @@ class ModelToComponentFactory:
|
|
877
1220
|
transformations = []
|
878
1221
|
if model.transformations:
|
879
1222
|
for transformation_model in model.transformations:
|
880
|
-
transformations.append(
|
1223
|
+
transformations.append(
|
1224
|
+
self._create_component_from_model(model=transformation_model, config=config)
|
1225
|
+
)
|
881
1226
|
retriever = self._create_component_from_model(
|
882
1227
|
model=model.retriever,
|
883
1228
|
config=config,
|
@@ -900,7 +1245,9 @@ class ModelToComponentFactory:
|
|
900
1245
|
state_transformations = []
|
901
1246
|
|
902
1247
|
if model.schema_loader:
|
903
|
-
schema_loader = self._create_component_from_model(
|
1248
|
+
schema_loader = self._create_component_from_model(
|
1249
|
+
model=model.schema_loader, config=config
|
1250
|
+
)
|
904
1251
|
else:
|
905
1252
|
options = model.parameters or {}
|
906
1253
|
if "name" not in options:
|
@@ -918,7 +1265,9 @@ class ModelToComponentFactory:
|
|
918
1265
|
parameters=model.parameters or {},
|
919
1266
|
)
|
920
1267
|
|
921
|
-
def _merge_stream_slicers(
|
1268
|
+
def _merge_stream_slicers(
|
1269
|
+
self, model: DeclarativeStreamModel, config: Config
|
1270
|
+
) -> Optional[StreamSlicer]:
|
922
1271
|
stream_slicer = None
|
923
1272
|
if (
|
924
1273
|
hasattr(model.retriever, "partition_router")
|
@@ -929,50 +1278,85 @@ class ModelToComponentFactory:
|
|
929
1278
|
|
930
1279
|
if isinstance(stream_slicer_model, list):
|
931
1280
|
stream_slicer = CartesianProductStreamSlicer(
|
932
|
-
[
|
1281
|
+
[
|
1282
|
+
self._create_component_from_model(model=slicer, config=config)
|
1283
|
+
for slicer in stream_slicer_model
|
1284
|
+
],
|
1285
|
+
parameters={},
|
933
1286
|
)
|
934
1287
|
else:
|
935
|
-
stream_slicer = self._create_component_from_model(
|
1288
|
+
stream_slicer = self._create_component_from_model(
|
1289
|
+
model=stream_slicer_model, config=config
|
1290
|
+
)
|
936
1291
|
|
937
1292
|
if model.incremental_sync and stream_slicer:
|
938
1293
|
incremental_sync_model = model.incremental_sync
|
939
|
-
if
|
940
|
-
|
941
|
-
|
1294
|
+
if (
|
1295
|
+
hasattr(incremental_sync_model, "global_substream_cursor")
|
1296
|
+
and incremental_sync_model.global_substream_cursor
|
1297
|
+
):
|
1298
|
+
cursor_component = self._create_component_from_model(
|
1299
|
+
model=incremental_sync_model, config=config
|
1300
|
+
)
|
1301
|
+
return GlobalSubstreamCursor(
|
1302
|
+
stream_cursor=cursor_component, partition_router=stream_slicer
|
1303
|
+
)
|
942
1304
|
else:
|
943
|
-
cursor_component = self._create_component_from_model(
|
1305
|
+
cursor_component = self._create_component_from_model(
|
1306
|
+
model=incremental_sync_model, config=config
|
1307
|
+
)
|
944
1308
|
return PerPartitionWithGlobalCursor(
|
945
1309
|
cursor_factory=CursorFactory(
|
946
|
-
lambda: self._create_component_from_model(
|
1310
|
+
lambda: self._create_component_from_model(
|
1311
|
+
model=incremental_sync_model, config=config
|
1312
|
+
),
|
947
1313
|
),
|
948
1314
|
partition_router=stream_slicer,
|
949
1315
|
stream_cursor=cursor_component,
|
950
1316
|
)
|
951
1317
|
elif model.incremental_sync:
|
952
|
-
return
|
1318
|
+
return (
|
1319
|
+
self._create_component_from_model(model=model.incremental_sync, config=config)
|
1320
|
+
if model.incremental_sync
|
1321
|
+
else None
|
1322
|
+
)
|
953
1323
|
elif stream_slicer:
|
954
1324
|
# For the Full-Refresh sub-streams, we use the nested `ChildPartitionResumableFullRefreshCursor`
|
955
1325
|
return PerPartitionCursor(
|
956
|
-
cursor_factory=CursorFactory(
|
1326
|
+
cursor_factory=CursorFactory(
|
1327
|
+
create_function=partial(ChildPartitionResumableFullRefreshCursor, {})
|
1328
|
+
),
|
957
1329
|
partition_router=stream_slicer,
|
958
1330
|
)
|
959
|
-
elif
|
1331
|
+
elif (
|
1332
|
+
hasattr(model.retriever, "paginator")
|
1333
|
+
and model.retriever.paginator
|
1334
|
+
and not stream_slicer
|
1335
|
+
):
|
960
1336
|
# For the regular Full-Refresh streams, we use the high level `ResumableFullRefreshCursor`
|
961
1337
|
return ResumableFullRefreshCursor(parameters={})
|
962
1338
|
else:
|
963
1339
|
return None
|
964
1340
|
|
965
|
-
def create_default_error_handler(
|
1341
|
+
def create_default_error_handler(
|
1342
|
+
self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any
|
1343
|
+
) -> DefaultErrorHandler:
|
966
1344
|
backoff_strategies = []
|
967
1345
|
if model.backoff_strategies:
|
968
1346
|
for backoff_strategy_model in model.backoff_strategies:
|
969
|
-
backoff_strategies.append(
|
1347
|
+
backoff_strategies.append(
|
1348
|
+
self._create_component_from_model(model=backoff_strategy_model, config=config)
|
1349
|
+
)
|
970
1350
|
|
971
1351
|
response_filters = []
|
972
1352
|
if model.response_filters:
|
973
1353
|
for response_filter_model in model.response_filters:
|
974
|
-
response_filters.append(
|
975
|
-
|
1354
|
+
response_filters.append(
|
1355
|
+
self._create_component_from_model(model=response_filter_model, config=config)
|
1356
|
+
)
|
1357
|
+
response_filters.append(
|
1358
|
+
HttpResponseFilter(config=config, parameters=model.parameters or {})
|
1359
|
+
)
|
976
1360
|
|
977
1361
|
return DefaultErrorHandler(
|
978
1362
|
backoff_strategies=backoff_strategies,
|
@@ -993,17 +1377,25 @@ class ModelToComponentFactory:
|
|
993
1377
|
) -> Union[DefaultPaginator, PaginatorTestReadDecorator]:
|
994
1378
|
if decoder:
|
995
1379
|
if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
|
996
|
-
raise ValueError(
|
1380
|
+
raise ValueError(
|
1381
|
+
f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
|
1382
|
+
)
|
997
1383
|
decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
|
998
1384
|
else:
|
999
1385
|
decoder_to_use = PaginationDecoderDecorator(decoder=JsonDecoder(parameters={}))
|
1000
1386
|
page_size_option = (
|
1001
|
-
self._create_component_from_model(model=model.page_size_option, config=config)
|
1387
|
+
self._create_component_from_model(model=model.page_size_option, config=config)
|
1388
|
+
if model.page_size_option
|
1389
|
+
else None
|
1002
1390
|
)
|
1003
1391
|
page_token_option = (
|
1004
|
-
self._create_component_from_model(model=model.page_token_option, config=config)
|
1392
|
+
self._create_component_from_model(model=model.page_token_option, config=config)
|
1393
|
+
if model.page_token_option
|
1394
|
+
else None
|
1395
|
+
)
|
1396
|
+
pagination_strategy = self._create_component_from_model(
|
1397
|
+
model=model.pagination_strategy, config=config, decoder=decoder_to_use
|
1005
1398
|
)
|
1006
|
-
pagination_strategy = self._create_component_from_model(model=model.pagination_strategy, config=config, decoder=decoder_to_use)
|
1007
1399
|
if cursor_used_for_stop_condition:
|
1008
1400
|
pagination_strategy = StopConditionPaginationStrategyDecorator(
|
1009
1401
|
pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition)
|
@@ -1022,29 +1414,55 @@ class ModelToComponentFactory:
|
|
1022
1414
|
return paginator
|
1023
1415
|
|
1024
1416
|
def create_dpath_extractor(
|
1025
|
-
self,
|
1417
|
+
self,
|
1418
|
+
model: DpathExtractorModel,
|
1419
|
+
config: Config,
|
1420
|
+
decoder: Optional[Decoder] = None,
|
1421
|
+
**kwargs: Any,
|
1026
1422
|
) -> DpathExtractor:
|
1027
1423
|
if decoder:
|
1028
1424
|
decoder_to_use = decoder
|
1029
1425
|
else:
|
1030
1426
|
decoder_to_use = JsonDecoder(parameters={})
|
1031
1427
|
model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path]
|
1032
|
-
return DpathExtractor(
|
1428
|
+
return DpathExtractor(
|
1429
|
+
decoder=decoder_to_use,
|
1430
|
+
field_path=model_field_path,
|
1431
|
+
config=config,
|
1432
|
+
parameters=model.parameters or {},
|
1433
|
+
)
|
1033
1434
|
|
1034
1435
|
@staticmethod
|
1035
|
-
def create_exponential_backoff_strategy(
|
1036
|
-
|
1436
|
+
def create_exponential_backoff_strategy(
|
1437
|
+
model: ExponentialBackoffStrategyModel, config: Config
|
1438
|
+
) -> ExponentialBackoffStrategy:
|
1439
|
+
return ExponentialBackoffStrategy(
|
1440
|
+
factor=model.factor or 5, parameters=model.parameters or {}, config=config
|
1441
|
+
)
|
1037
1442
|
|
1038
|
-
def create_http_requester(
|
1443
|
+
def create_http_requester(
|
1444
|
+
self, model: HttpRequesterModel, decoder: Decoder, config: Config, *, name: str
|
1445
|
+
) -> HttpRequester:
|
1039
1446
|
authenticator = (
|
1040
|
-
self._create_component_from_model(
|
1447
|
+
self._create_component_from_model(
|
1448
|
+
model=model.authenticator,
|
1449
|
+
config=config,
|
1450
|
+
url_base=model.url_base,
|
1451
|
+
name=name,
|
1452
|
+
decoder=decoder,
|
1453
|
+
)
|
1041
1454
|
if model.authenticator
|
1042
1455
|
else None
|
1043
1456
|
)
|
1044
1457
|
error_handler = (
|
1045
1458
|
self._create_component_from_model(model=model.error_handler, config=config)
|
1046
1459
|
if model.error_handler
|
1047
|
-
else DefaultErrorHandler(
|
1460
|
+
else DefaultErrorHandler(
|
1461
|
+
backoff_strategies=[],
|
1462
|
+
response_filters=[],
|
1463
|
+
config=config,
|
1464
|
+
parameters=model.parameters or {},
|
1465
|
+
)
|
1048
1466
|
)
|
1049
1467
|
|
1050
1468
|
request_options_provider = InterpolatedRequestOptionsProvider(
|
@@ -1079,7 +1497,9 @@ class ModelToComponentFactory:
|
|
1079
1497
|
)
|
1080
1498
|
|
1081
1499
|
@staticmethod
|
1082
|
-
def create_http_response_filter(
|
1500
|
+
def create_http_response_filter(
|
1501
|
+
model: HttpResponseFilterModel, config: Config, **kwargs: Any
|
1502
|
+
) -> HttpResponseFilter:
|
1083
1503
|
if model.action:
|
1084
1504
|
action = ResponseAction(model.action.value)
|
1085
1505
|
else:
|
@@ -1103,7 +1523,9 @@ class ModelToComponentFactory:
|
|
1103
1523
|
)
|
1104
1524
|
|
1105
1525
|
@staticmethod
|
1106
|
-
def create_inline_schema_loader(
|
1526
|
+
def create_inline_schema_loader(
|
1527
|
+
model: InlineSchemaLoaderModel, config: Config, **kwargs: Any
|
1528
|
+
) -> InlineSchemaLoader:
|
1107
1529
|
return InlineSchemaLoader(schema=model.schema_ or {}, parameters={})
|
1108
1530
|
|
1109
1531
|
@staticmethod
|
@@ -1111,11 +1533,15 @@ class ModelToComponentFactory:
|
|
1111
1533
|
return JsonDecoder(parameters={})
|
1112
1534
|
|
1113
1535
|
@staticmethod
|
1114
|
-
def create_jsonl_decoder(
|
1536
|
+
def create_jsonl_decoder(
|
1537
|
+
model: JsonlDecoderModel, config: Config, **kwargs: Any
|
1538
|
+
) -> JsonlDecoder:
|
1115
1539
|
return JsonlDecoder(parameters={})
|
1116
1540
|
|
1117
1541
|
@staticmethod
|
1118
|
-
def create_iterable_decoder(
|
1542
|
+
def create_iterable_decoder(
|
1543
|
+
model: IterableDecoderModel, config: Config, **kwargs: Any
|
1544
|
+
) -> IterableDecoder:
|
1119
1545
|
return IterableDecoder(parameters={})
|
1120
1546
|
|
1121
1547
|
@staticmethod
|
@@ -1123,11 +1549,17 @@ class ModelToComponentFactory:
|
|
1123
1549
|
return XmlDecoder(parameters={})
|
1124
1550
|
|
1125
1551
|
@staticmethod
|
1126
|
-
def create_json_file_schema_loader(
|
1127
|
-
|
1552
|
+
def create_json_file_schema_loader(
|
1553
|
+
model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any
|
1554
|
+
) -> JsonFileSchemaLoader:
|
1555
|
+
return JsonFileSchemaLoader(
|
1556
|
+
file_path=model.file_path or "", config=config, parameters=model.parameters or {}
|
1557
|
+
)
|
1128
1558
|
|
1129
1559
|
@staticmethod
|
1130
|
-
def create_jwt_authenticator(
|
1560
|
+
def create_jwt_authenticator(
|
1561
|
+
model: JwtAuthenticatorModel, config: Config, **kwargs: Any
|
1562
|
+
) -> JwtAuthenticator:
|
1131
1563
|
jwt_headers = model.jwt_headers or JwtHeadersModel(kid=None, typ="JWT", cty=None)
|
1132
1564
|
jwt_payload = model.jwt_payload or JwtPayloadModel(iss=None, sub=None, aud=None)
|
1133
1565
|
return JwtAuthenticator(
|
@@ -1149,7 +1581,9 @@ class ModelToComponentFactory:
|
|
1149
1581
|
)
|
1150
1582
|
|
1151
1583
|
@staticmethod
|
1152
|
-
def create_list_partition_router(
|
1584
|
+
def create_list_partition_router(
|
1585
|
+
model: ListPartitionRouterModel, config: Config, **kwargs: Any
|
1586
|
+
) -> ListPartitionRouter:
|
1153
1587
|
request_option = (
|
1154
1588
|
RequestOption(
|
1155
1589
|
inject_into=RequestOptionType(model.request_option.inject_into.value),
|
@@ -1168,7 +1602,9 @@ class ModelToComponentFactory:
|
|
1168
1602
|
)
|
1169
1603
|
|
1170
1604
|
@staticmethod
|
1171
|
-
def create_min_max_datetime(
|
1605
|
+
def create_min_max_datetime(
|
1606
|
+
model: MinMaxDatetimeModel, config: Config, **kwargs: Any
|
1607
|
+
) -> MinMaxDatetime:
|
1172
1608
|
return MinMaxDatetime(
|
1173
1609
|
datetime=model.datetime,
|
1174
1610
|
datetime_format=model.datetime_format or "",
|
@@ -1182,29 +1618,43 @@ class ModelToComponentFactory:
|
|
1182
1618
|
return NoAuth(parameters=model.parameters or {})
|
1183
1619
|
|
1184
1620
|
@staticmethod
|
1185
|
-
def create_no_pagination(
|
1621
|
+
def create_no_pagination(
|
1622
|
+
model: NoPaginationModel, config: Config, **kwargs: Any
|
1623
|
+
) -> NoPagination:
|
1186
1624
|
return NoPagination(parameters={})
|
1187
1625
|
|
1188
|
-
def create_oauth_authenticator(
|
1626
|
+
def create_oauth_authenticator(
|
1627
|
+
self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any
|
1628
|
+
) -> DeclarativeOauth2Authenticator:
|
1189
1629
|
if model.refresh_token_updater:
|
1190
1630
|
# ignore type error because fixing it would have a lot of dependencies, revisit later
|
1191
1631
|
return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore
|
1192
1632
|
config,
|
1193
|
-
InterpolatedString.create(
|
1633
|
+
InterpolatedString.create(
|
1634
|
+
model.token_refresh_endpoint, parameters=model.parameters or {}
|
1635
|
+
).eval(config),
|
1194
1636
|
access_token_name=InterpolatedString.create(
|
1195
1637
|
model.access_token_name or "access_token", parameters=model.parameters or {}
|
1196
1638
|
).eval(config),
|
1197
1639
|
refresh_token_name=model.refresh_token_updater.refresh_token_name,
|
1198
|
-
expires_in_name=InterpolatedString.create(
|
1199
|
-
|
1200
|
-
),
|
1201
|
-
client_id=InterpolatedString.create(
|
1202
|
-
|
1640
|
+
expires_in_name=InterpolatedString.create(
|
1641
|
+
model.expires_in_name or "expires_in", parameters=model.parameters or {}
|
1642
|
+
).eval(config),
|
1643
|
+
client_id=InterpolatedString.create(
|
1644
|
+
model.client_id, parameters=model.parameters or {}
|
1645
|
+
).eval(config),
|
1646
|
+
client_secret=InterpolatedString.create(
|
1647
|
+
model.client_secret, parameters=model.parameters or {}
|
1648
|
+
).eval(config),
|
1203
1649
|
access_token_config_path=model.refresh_token_updater.access_token_config_path,
|
1204
1650
|
refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path,
|
1205
1651
|
token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path,
|
1206
|
-
grant_type=InterpolatedString.create(
|
1207
|
-
|
1652
|
+
grant_type=InterpolatedString.create(
|
1653
|
+
model.grant_type or "refresh_token", parameters=model.parameters or {}
|
1654
|
+
).eval(config),
|
1655
|
+
refresh_request_body=InterpolatedMapping(
|
1656
|
+
model.refresh_request_body or {}, parameters=model.parameters or {}
|
1657
|
+
).eval(config),
|
1208
1658
|
scopes=model.scopes,
|
1209
1659
|
token_expiry_date_format=model.token_expiry_date_format,
|
1210
1660
|
message_repository=self._message_repository,
|
@@ -1232,7 +1682,9 @@ class ModelToComponentFactory:
|
|
1232
1682
|
)
|
1233
1683
|
|
1234
1684
|
@staticmethod
|
1235
|
-
def create_offset_increment(
|
1685
|
+
def create_offset_increment(
|
1686
|
+
model: OffsetIncrementModel, config: Config, decoder: Decoder, **kwargs: Any
|
1687
|
+
) -> OffsetIncrement:
|
1236
1688
|
if isinstance(decoder, PaginationDecoderDecorator):
|
1237
1689
|
if not isinstance(decoder.decoder, (JsonDecoder, XmlDecoder)):
|
1238
1690
|
raise ValueError(
|
@@ -1241,7 +1693,9 @@ class ModelToComponentFactory:
|
|
1241
1693
|
decoder_to_use = decoder
|
1242
1694
|
else:
|
1243
1695
|
if not isinstance(decoder, (JsonDecoder, XmlDecoder)):
|
1244
|
-
raise ValueError(
|
1696
|
+
raise ValueError(
|
1697
|
+
f"Provided decoder of {type(decoder)=} is not supported. Please set JsonDecoder or XmlDecoder instead."
|
1698
|
+
)
|
1245
1699
|
decoder_to_use = PaginationDecoderDecorator(decoder=decoder)
|
1246
1700
|
return OffsetIncrement(
|
1247
1701
|
page_size=model.page_size,
|
@@ -1252,7 +1706,9 @@ class ModelToComponentFactory:
|
|
1252
1706
|
)
|
1253
1707
|
|
1254
1708
|
@staticmethod
|
1255
|
-
def create_page_increment(
|
1709
|
+
def create_page_increment(
|
1710
|
+
model: PageIncrementModel, config: Config, **kwargs: Any
|
1711
|
+
) -> PageIncrement:
|
1256
1712
|
return PageIncrement(
|
1257
1713
|
page_size=model.page_size,
|
1258
1714
|
config=config,
|
@@ -1261,9 +1717,15 @@ class ModelToComponentFactory:
|
|
1261
1717
|
parameters=model.parameters or {},
|
1262
1718
|
)
|
1263
1719
|
|
1264
|
-
def create_parent_stream_config(
|
1720
|
+
def create_parent_stream_config(
|
1721
|
+
self, model: ParentStreamConfigModel, config: Config, **kwargs: Any
|
1722
|
+
) -> ParentStreamConfig:
|
1265
1723
|
declarative_stream = self._create_component_from_model(model.stream, config=config)
|
1266
|
-
request_option =
|
1724
|
+
request_option = (
|
1725
|
+
self._create_component_from_model(model.request_option, config=config)
|
1726
|
+
if model.request_option
|
1727
|
+
else None
|
1728
|
+
)
|
1267
1729
|
return ParentStreamConfig(
|
1268
1730
|
parent_key=model.parent_key,
|
1269
1731
|
request_option=request_option,
|
@@ -1276,15 +1738,21 @@ class ModelToComponentFactory:
|
|
1276
1738
|
)
|
1277
1739
|
|
1278
1740
|
@staticmethod
|
1279
|
-
def create_record_filter(
|
1280
|
-
|
1741
|
+
def create_record_filter(
|
1742
|
+
model: RecordFilterModel, config: Config, **kwargs: Any
|
1743
|
+
) -> RecordFilter:
|
1744
|
+
return RecordFilter(
|
1745
|
+
condition=model.condition or "", config=config, parameters=model.parameters or {}
|
1746
|
+
)
|
1281
1747
|
|
1282
1748
|
@staticmethod
|
1283
1749
|
def create_request_path(model: RequestPathModel, config: Config, **kwargs: Any) -> RequestPath:
|
1284
1750
|
return RequestPath(parameters={})
|
1285
1751
|
|
1286
1752
|
@staticmethod
|
1287
|
-
def create_request_option(
|
1753
|
+
def create_request_option(
|
1754
|
+
model: RequestOptionModel, config: Config, **kwargs: Any
|
1755
|
+
) -> RequestOption:
|
1288
1756
|
inject_into = RequestOptionType(model.inject_into.value)
|
1289
1757
|
return RequestOption(field_name=model.field_name, inject_into=inject_into, parameters={})
|
1290
1758
|
|
@@ -1299,16 +1767,26 @@ class ModelToComponentFactory:
|
|
1299
1767
|
**kwargs: Any,
|
1300
1768
|
) -> RecordSelector:
|
1301
1769
|
assert model.schema_normalization is not None # for mypy
|
1302
|
-
extractor = self._create_component_from_model(
|
1303
|
-
|
1770
|
+
extractor = self._create_component_from_model(
|
1771
|
+
model=model.extractor, decoder=decoder, config=config
|
1772
|
+
)
|
1773
|
+
record_filter = (
|
1774
|
+
self._create_component_from_model(model.record_filter, config=config)
|
1775
|
+
if model.record_filter
|
1776
|
+
else None
|
1777
|
+
)
|
1304
1778
|
if client_side_incremental_sync:
|
1305
1779
|
record_filter = ClientSideIncrementalRecordFilterDecorator(
|
1306
1780
|
config=config,
|
1307
1781
|
parameters=model.parameters,
|
1308
|
-
condition=model.record_filter.condition
|
1782
|
+
condition=model.record_filter.condition
|
1783
|
+
if (model.record_filter and hasattr(model.record_filter, "condition"))
|
1784
|
+
else None,
|
1309
1785
|
**client_side_incremental_sync,
|
1310
1786
|
)
|
1311
|
-
schema_normalization = TypeTransformer(
|
1787
|
+
schema_normalization = TypeTransformer(
|
1788
|
+
SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]
|
1789
|
+
)
|
1312
1790
|
|
1313
1791
|
return RecordSelector(
|
1314
1792
|
extractor=extractor,
|
@@ -1320,11 +1798,20 @@ class ModelToComponentFactory:
|
|
1320
1798
|
)
|
1321
1799
|
|
1322
1800
|
@staticmethod
|
1323
|
-
def create_remove_fields(
|
1324
|
-
|
1325
|
-
|
1326
|
-
|
1327
|
-
|
1801
|
+
def create_remove_fields(
|
1802
|
+
model: RemoveFieldsModel, config: Config, **kwargs: Any
|
1803
|
+
) -> RemoveFields:
|
1804
|
+
return RemoveFields(
|
1805
|
+
field_pointers=model.field_pointers, condition=model.condition or "", parameters={}
|
1806
|
+
)
|
1807
|
+
|
1808
|
+
def create_selective_authenticator(
|
1809
|
+
self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any
|
1810
|
+
) -> DeclarativeAuthenticator:
|
1811
|
+
authenticators = {
|
1812
|
+
name: self._create_component_from_model(model=auth, config=config)
|
1813
|
+
for name, auth in model.authenticators.items()
|
1814
|
+
}
|
1328
1815
|
# SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error
|
1329
1816
|
return SelectiveAuthenticator( # type: ignore[abstract]
|
1330
1817
|
config=config,
|
@@ -1363,8 +1850,14 @@ class ModelToComponentFactory:
|
|
1363
1850
|
client_side_incremental_sync: Optional[Dict[str, Any]] = None,
|
1364
1851
|
transformations: List[RecordTransformation],
|
1365
1852
|
) -> SimpleRetriever:
|
1366
|
-
decoder =
|
1367
|
-
|
1853
|
+
decoder = (
|
1854
|
+
self._create_component_from_model(model=model.decoder, config=config)
|
1855
|
+
if model.decoder
|
1856
|
+
else JsonDecoder(parameters={})
|
1857
|
+
)
|
1858
|
+
requester = self._create_component_from_model(
|
1859
|
+
model=model.requester, decoder=decoder, config=config, name=name
|
1860
|
+
)
|
1368
1861
|
record_selector = self._create_component_from_model(
|
1369
1862
|
model=model.record_selector,
|
1370
1863
|
config=config,
|
@@ -1372,12 +1865,19 @@ class ModelToComponentFactory:
|
|
1372
1865
|
transformations=transformations,
|
1373
1866
|
client_side_incremental_sync=client_side_incremental_sync,
|
1374
1867
|
)
|
1375
|
-
url_base =
|
1868
|
+
url_base = (
|
1869
|
+
model.requester.url_base
|
1870
|
+
if hasattr(model.requester, "url_base")
|
1871
|
+
else requester.get_url_base()
|
1872
|
+
)
|
1376
1873
|
|
1377
1874
|
# Define cursor only if per partition or common incremental support is needed
|
1378
1875
|
cursor = stream_slicer if isinstance(stream_slicer, DeclarativeCursor) else None
|
1379
1876
|
|
1380
|
-
if
|
1877
|
+
if (
|
1878
|
+
not isinstance(stream_slicer, DatetimeBasedCursor)
|
1879
|
+
or type(stream_slicer) is not DatetimeBasedCursor
|
1880
|
+
):
|
1381
1881
|
# Many of the custom component implementations of DatetimeBasedCursor override get_request_params() (or other methods).
|
1382
1882
|
# Because we're decoupling RequestOptionsProvider from the Cursor, custom components will eventually need to reimplement
|
1383
1883
|
# their own RequestOptionsProvider. However, right now the existing StreamSlicer/Cursor still can act as the SimpleRetriever's
|
@@ -1401,7 +1901,9 @@ class ModelToComponentFactory:
|
|
1401
1901
|
else NoPagination(parameters={})
|
1402
1902
|
)
|
1403
1903
|
|
1404
|
-
ignore_stream_slicer_parameters_on_paginated_requests =
|
1904
|
+
ignore_stream_slicer_parameters_on_paginated_requests = (
|
1905
|
+
model.ignore_stream_slicer_parameters_on_paginated_requests or False
|
1906
|
+
)
|
1405
1907
|
|
1406
1908
|
if self._limit_slices_fetched or self._emit_connector_builder_messages:
|
1407
1909
|
return SimpleRetrieverTestReadDecorator(
|
@@ -1468,14 +1970,19 @@ class ModelToComponentFactory:
|
|
1468
1970
|
config: Config,
|
1469
1971
|
*,
|
1470
1972
|
name: str,
|
1471
|
-
primary_key: Optional[
|
1973
|
+
primary_key: Optional[
|
1974
|
+
Union[str, List[str], List[List[str]]]
|
1975
|
+
], # this seems to be needed to match create_simple_retriever
|
1472
1976
|
stream_slicer: Optional[StreamSlicer],
|
1473
1977
|
client_side_incremental_sync: Optional[Dict[str, Any]] = None,
|
1474
1978
|
transformations: List[RecordTransformation],
|
1475
1979
|
**kwargs: Any,
|
1476
1980
|
) -> AsyncRetriever:
|
1477
|
-
|
1478
|
-
|
1981
|
+
decoder = (
|
1982
|
+
self._create_component_from_model(model=model.decoder, config=config)
|
1983
|
+
if model.decoder
|
1984
|
+
else JsonDecoder(parameters={})
|
1985
|
+
)
|
1479
1986
|
record_selector = self._create_component_from_model(
|
1480
1987
|
model=model.record_selector,
|
1481
1988
|
config=config,
|
@@ -1485,14 +1992,23 @@ class ModelToComponentFactory:
|
|
1485
1992
|
)
|
1486
1993
|
stream_slicer = stream_slicer or SinglePartitionRouter(parameters={})
|
1487
1994
|
creation_requester = self._create_component_from_model(
|
1488
|
-
model=model.creation_requester,
|
1995
|
+
model=model.creation_requester,
|
1996
|
+
decoder=decoder,
|
1997
|
+
config=config,
|
1998
|
+
name=f"job creation - {name}",
|
1489
1999
|
)
|
1490
2000
|
polling_requester = self._create_component_from_model(
|
1491
|
-
model=model.polling_requester,
|
2001
|
+
model=model.polling_requester,
|
2002
|
+
decoder=decoder,
|
2003
|
+
config=config,
|
2004
|
+
name=f"job polling - {name}",
|
1492
2005
|
)
|
1493
2006
|
job_download_components_name = f"job download - {name}"
|
1494
2007
|
download_requester = self._create_component_from_model(
|
1495
|
-
model=model.download_requester,
|
2008
|
+
model=model.download_requester,
|
2009
|
+
decoder=decoder,
|
2010
|
+
config=config,
|
2011
|
+
name=job_download_components_name,
|
1496
2012
|
)
|
1497
2013
|
download_retriever = SimpleRetriever(
|
1498
2014
|
requester=download_requester,
|
@@ -1507,7 +2023,9 @@ class ModelToComponentFactory:
|
|
1507
2023
|
primary_key=None,
|
1508
2024
|
name=job_download_components_name,
|
1509
2025
|
paginator=(
|
1510
|
-
self._create_component_from_model(
|
2026
|
+
self._create_component_from_model(
|
2027
|
+
model=model.download_paginator, decoder=decoder, config=config, url_base=""
|
2028
|
+
)
|
1511
2029
|
if model.download_paginator
|
1512
2030
|
else NoPagination(parameters={})
|
1513
2031
|
),
|
@@ -1515,17 +2033,31 @@ class ModelToComponentFactory:
|
|
1515
2033
|
parameters={},
|
1516
2034
|
)
|
1517
2035
|
abort_requester = (
|
1518
|
-
self._create_component_from_model(
|
2036
|
+
self._create_component_from_model(
|
2037
|
+
model=model.abort_requester,
|
2038
|
+
decoder=decoder,
|
2039
|
+
config=config,
|
2040
|
+
name=f"job abort - {name}",
|
2041
|
+
)
|
1519
2042
|
if model.abort_requester
|
1520
2043
|
else None
|
1521
2044
|
)
|
1522
2045
|
delete_requester = (
|
1523
|
-
self._create_component_from_model(
|
2046
|
+
self._create_component_from_model(
|
2047
|
+
model=model.delete_requester,
|
2048
|
+
decoder=decoder,
|
2049
|
+
config=config,
|
2050
|
+
name=f"job delete - {name}",
|
2051
|
+
)
|
1524
2052
|
if model.delete_requester
|
1525
2053
|
else None
|
1526
2054
|
)
|
1527
|
-
status_extractor = self._create_component_from_model(
|
1528
|
-
|
2055
|
+
status_extractor = self._create_component_from_model(
|
2056
|
+
model=model.status_extractor, decoder=decoder, config=config, name=name
|
2057
|
+
)
|
2058
|
+
urls_extractor = self._create_component_from_model(
|
2059
|
+
model=model.urls_extractor, decoder=decoder, config=config, name=name
|
2060
|
+
)
|
1529
2061
|
job_repository: AsyncJobRepository = AsyncHttpJobRepository(
|
1530
2062
|
creation_requester=creation_requester,
|
1531
2063
|
polling_requester=polling_requester,
|
@@ -1541,7 +2073,9 @@ class ModelToComponentFactory:
|
|
1541
2073
|
job_orchestrator_factory=lambda stream_slices: AsyncJobOrchestrator(
|
1542
2074
|
job_repository,
|
1543
2075
|
stream_slices,
|
1544
|
-
JobTracker(
|
2076
|
+
JobTracker(
|
2077
|
+
1
|
2078
|
+
), # FIXME eventually make the number of concurrent jobs in the API configurable. Until then, we limit to 1
|
1545
2079
|
self._message_repository,
|
1546
2080
|
has_bulk_parent=False, # FIXME work would need to be done here in order to detect if a stream as a parent stream that is bulk
|
1547
2081
|
),
|
@@ -1567,14 +2101,22 @@ class ModelToComponentFactory:
|
|
1567
2101
|
if model.parent_stream_configs:
|
1568
2102
|
parent_stream_configs.extend(
|
1569
2103
|
[
|
1570
|
-
self._create_message_repository_substream_wrapper(
|
2104
|
+
self._create_message_repository_substream_wrapper(
|
2105
|
+
model=parent_stream_config, config=config
|
2106
|
+
)
|
1571
2107
|
for parent_stream_config in model.parent_stream_configs
|
1572
2108
|
]
|
1573
2109
|
)
|
1574
2110
|
|
1575
|
-
return SubstreamPartitionRouter(
|
2111
|
+
return SubstreamPartitionRouter(
|
2112
|
+
parent_stream_configs=parent_stream_configs,
|
2113
|
+
parameters=model.parameters or {},
|
2114
|
+
config=config,
|
2115
|
+
)
|
1576
2116
|
|
1577
|
-
def _create_message_repository_substream_wrapper(
|
2117
|
+
def _create_message_repository_substream_wrapper(
|
2118
|
+
self, model: ParentStreamConfigModel, config: Config
|
2119
|
+
) -> Any:
|
1578
2120
|
substream_factory = ModelToComponentFactory(
|
1579
2121
|
limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice,
|
1580
2122
|
limit_slices_fetched=self._limit_slices_fetched,
|
@@ -1590,13 +2132,17 @@ class ModelToComponentFactory:
|
|
1590
2132
|
return substream_factory._create_component_from_model(model=model, config=config)
|
1591
2133
|
|
1592
2134
|
@staticmethod
|
1593
|
-
def create_wait_time_from_header(
|
2135
|
+
def create_wait_time_from_header(
|
2136
|
+
model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any
|
2137
|
+
) -> WaitTimeFromHeaderBackoffStrategy:
|
1594
2138
|
return WaitTimeFromHeaderBackoffStrategy(
|
1595
2139
|
header=model.header,
|
1596
2140
|
parameters=model.parameters or {},
|
1597
2141
|
config=config,
|
1598
2142
|
regex=model.regex,
|
1599
|
-
max_waiting_time_in_seconds=model.max_waiting_time_in_seconds
|
2143
|
+
max_waiting_time_in_seconds=model.max_waiting_time_in_seconds
|
2144
|
+
if model.max_waiting_time_in_seconds is not None
|
2145
|
+
else None,
|
1600
2146
|
)
|
1601
2147
|
|
1602
2148
|
@staticmethod
|
@@ -1604,7 +2150,11 @@ class ModelToComponentFactory:
|
|
1604
2150
|
model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any
|
1605
2151
|
) -> WaitUntilTimeFromHeaderBackoffStrategy:
|
1606
2152
|
return WaitUntilTimeFromHeaderBackoffStrategy(
|
1607
|
-
header=model.header,
|
2153
|
+
header=model.header,
|
2154
|
+
parameters=model.parameters or {},
|
2155
|
+
config=config,
|
2156
|
+
min_wait=model.min_wait,
|
2157
|
+
regex=model.regex,
|
1608
2158
|
)
|
1609
2159
|
|
1610
2160
|
def get_message_repository(self) -> MessageRepository:
|