airbyte-cdk 6.5.3rc2__py3-none-any.whl → 6.5.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/__init__.py +17 -2
- airbyte_cdk/config_observation.py +10 -3
- airbyte_cdk/connector.py +19 -9
- airbyte_cdk/connector_builder/connector_builder_handler.py +28 -8
- airbyte_cdk/connector_builder/main.py +26 -6
- airbyte_cdk/connector_builder/message_grouper.py +95 -25
- airbyte_cdk/destinations/destination.py +47 -14
- airbyte_cdk/destinations/vector_db_based/config.py +36 -14
- airbyte_cdk/destinations/vector_db_based/document_processor.py +49 -11
- airbyte_cdk/destinations/vector_db_based/embedder.py +52 -11
- airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
- airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +15 -4
- airbyte_cdk/entrypoint.py +82 -26
- airbyte_cdk/exception_handler.py +13 -3
- airbyte_cdk/logger.py +10 -2
- airbyte_cdk/models/airbyte_protocol.py +11 -5
- airbyte_cdk/models/airbyte_protocol_serializers.py +9 -3
- airbyte_cdk/models/well_known_types.py +1 -1
- airbyte_cdk/sources/abstract_source.py +63 -17
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +47 -14
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +25 -7
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +27 -6
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +9 -3
- airbyte_cdk/sources/connector_state_manager.py +32 -10
- airbyte_cdk/sources/declarative/async_job/job.py +3 -1
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +68 -14
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +24 -6
- airbyte_cdk/sources/declarative/async_job/repository.py +3 -1
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +27 -7
- airbyte_cdk/sources/declarative/auth/oauth.py +35 -11
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/token.py +25 -8
- airbyte_cdk/sources/declarative/checks/check_stream.py +12 -4
- airbyte_cdk/sources/declarative/checks/connection_checker.py +3 -1
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +11 -3
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +106 -50
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +20 -6
- airbyte_cdk/sources/declarative/declarative_source.py +3 -1
- airbyte_cdk/sources/declarative/declarative_stream.py +27 -6
- airbyte_cdk/sources/declarative/decoders/decoder.py +3 -1
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +3 -1
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +3 -1
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +6 -2
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +6 -2
- airbyte_cdk/sources/declarative/extractors/record_filter.py +24 -7
- airbyte_cdk/sources/declarative/extractors/record_selector.py +10 -3
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +15 -5
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +96 -31
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +22 -8
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +46 -15
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +19 -5
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +3 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +20 -2
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +5 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +10 -3
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +6 -2
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +7 -1
- airbyte_cdk/sources/declarative/interpolation/jinja.py +6 -2
- airbyte_cdk/sources/declarative/interpolation/macros.py +19 -4
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +106 -24
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +7 -2
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +656 -678
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +13 -4
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +9 -2
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +782 -232
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +29 -7
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +25 -7
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +54 -15
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +6 -2
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +17 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +15 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +18 -8
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +16 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +51 -14
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +29 -8
- airbyte_cdk/sources/declarative/requesters/http_requester.py +58 -16
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +49 -14
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +3 -1
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +3 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +17 -5
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +24 -7
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +9 -3
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +3 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +6 -2
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +19 -6
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +3 -1
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +21 -7
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +18 -6
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +27 -8
- airbyte_cdk/sources/declarative/requesters/requester.py +3 -1
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +12 -5
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +105 -24
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +3 -1
- airbyte_cdk/sources/declarative/spec/spec.py +8 -2
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +3 -1
- airbyte_cdk/sources/declarative/transformations/add_fields.py +12 -3
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +6 -2
- airbyte_cdk/sources/declarative/types.py +8 -1
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +3 -1
- airbyte_cdk/sources/embedded/base_integration.py +14 -4
- airbyte_cdk/sources/embedded/catalog.py +16 -4
- airbyte_cdk/sources/embedded/runner.py +19 -3
- airbyte_cdk/sources/embedded/tools.py +3 -1
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +12 -4
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +27 -7
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +12 -6
- airbyte_cdk/sources/file_based/config/csv_format.py +21 -9
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +6 -2
- airbyte_cdk/sources/file_based/config/unstructured_format.py +10 -3
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
- airbyte_cdk/sources/file_based/exceptions.py +13 -15
- airbyte_cdk/sources/file_based/file_based_source.py +82 -24
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +16 -5
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +58 -17
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +89 -26
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +25 -7
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +8 -2
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +20 -6
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +57 -16
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +64 -15
- airbyte_cdk/sources/file_based/schema_helpers.py +33 -10
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +33 -10
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +47 -11
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +13 -22
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +53 -17
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +17 -5
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +26 -9
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +67 -21
- airbyte_cdk/sources/http_logger.py +5 -1
- airbyte_cdk/sources/message/repository.py +18 -4
- airbyte_cdk/sources/source.py +17 -7
- airbyte_cdk/sources/streams/availability_strategy.py +9 -3
- airbyte_cdk/sources/streams/call_rate.py +63 -19
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +31 -7
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +6 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +77 -22
- airbyte_cdk/sources/streams/concurrent/cursor.py +56 -20
- airbyte_cdk/sources/streams/concurrent/default_stream.py +9 -2
- airbyte_cdk/sources/streams/concurrent/helpers.py +6 -2
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +9 -2
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +4 -1
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +10 -2
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +6 -2
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +25 -10
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +32 -16
- airbyte_cdk/sources/streams/core.py +77 -22
- airbyte_cdk/sources/streams/http/availability_strategy.py +3 -1
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +4 -1
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +3 -1
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +16 -5
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +9 -3
- airbyte_cdk/sources/streams/http/exceptions.py +2 -2
- airbyte_cdk/sources/streams/http/http.py +133 -33
- airbyte_cdk/sources/streams/http/http_client.py +91 -29
- airbyte_cdk/sources/streams/http/rate_limiting.py +23 -7
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +19 -6
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +38 -11
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
- airbyte_cdk/sources/types.py +5 -1
- airbyte_cdk/sources/utils/record_helper.py +12 -3
- airbyte_cdk/sources/utils/schema_helpers.py +9 -3
- airbyte_cdk/sources/utils/slice_logger.py +4 -1
- airbyte_cdk/sources/utils/transform.py +24 -9
- airbyte_cdk/sql/exceptions.py +19 -6
- airbyte_cdk/sql/secrets.py +3 -1
- airbyte_cdk/sql/shared/catalog_providers.py +13 -4
- airbyte_cdk/sql/shared/sql_processor.py +44 -14
- airbyte_cdk/test/catalog_builder.py +19 -8
- airbyte_cdk/test/entrypoint_wrapper.py +27 -8
- airbyte_cdk/test/mock_http/mocker.py +41 -11
- airbyte_cdk/test/mock_http/request.py +9 -3
- airbyte_cdk/test/mock_http/response.py +3 -1
- airbyte_cdk/test/mock_http/response_builder.py +29 -7
- airbyte_cdk/test/state_builder.py +10 -2
- airbyte_cdk/test/utils/data.py +6 -2
- airbyte_cdk/test/utils/http_mocking.py +3 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +3 -1
- airbyte_cdk/utils/analytics_message.py +10 -2
- airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
- airbyte_cdk/utils/mapping_helpers.py +3 -1
- airbyte_cdk/utils/message_utils.py +11 -4
- airbyte_cdk/utils/print_buffer.py +6 -1
- airbyte_cdk/utils/schema_inferrer.py +30 -9
- airbyte_cdk/utils/spec_schema_transformations.py +3 -1
- airbyte_cdk/utils/traced_exception.py +35 -9
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/METADATA +7 -6
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/RECORD +198 -198
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.5.5.dist-info}/WHEEL +0 -0
@@ -4,321 +4,320 @@
|
|
4
4
|
from __future__ import annotations
|
5
5
|
|
6
6
|
from enum import Enum
|
7
|
-
from typing import Any, Dict, List, Optional, Union
|
7
|
+
from typing import Any, Dict, List, Literal, Optional, Union
|
8
8
|
|
9
9
|
from pydantic.v1 import BaseModel, Extra, Field
|
10
|
-
from typing_extensions import Literal
|
11
10
|
|
12
11
|
|
13
12
|
class AuthFlowType(Enum):
|
14
|
-
oauth2_0 =
|
15
|
-
oauth1_0 =
|
13
|
+
oauth2_0 = "oauth2.0"
|
14
|
+
oauth1_0 = "oauth1.0"
|
16
15
|
|
17
16
|
|
18
17
|
class BasicHttpAuthenticator(BaseModel):
|
19
|
-
type: Literal[
|
18
|
+
type: Literal["BasicHttpAuthenticator"]
|
20
19
|
username: str = Field(
|
21
20
|
...,
|
22
|
-
description=
|
21
|
+
description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.",
|
23
22
|
examples=["{{ config['username'] }}", "{{ config['api_key'] }}"],
|
24
|
-
title=
|
23
|
+
title="Username",
|
25
24
|
)
|
26
25
|
password: Optional[str] = Field(
|
27
|
-
|
28
|
-
description=
|
29
|
-
examples=["{{ config['password'] }}",
|
30
|
-
title=
|
26
|
+
"",
|
27
|
+
description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.",
|
28
|
+
examples=["{{ config['password'] }}", ""],
|
29
|
+
title="Password",
|
31
30
|
)
|
32
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
31
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
33
32
|
|
34
33
|
|
35
34
|
class BearerAuthenticator(BaseModel):
|
36
|
-
type: Literal[
|
35
|
+
type: Literal["BearerAuthenticator"]
|
37
36
|
api_token: str = Field(
|
38
37
|
...,
|
39
|
-
description=
|
38
|
+
description="Token to inject as request header for authenticating with the API.",
|
40
39
|
examples=["{{ config['api_key'] }}", "{{ config['token'] }}"],
|
41
|
-
title=
|
40
|
+
title="Bearer Token",
|
42
41
|
)
|
43
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
42
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
44
43
|
|
45
44
|
|
46
45
|
class CheckStream(BaseModel):
|
47
|
-
type: Literal[
|
46
|
+
type: Literal["CheckStream"]
|
48
47
|
stream_names: List[str] = Field(
|
49
48
|
...,
|
50
|
-
description=
|
51
|
-
examples=[[
|
52
|
-
title=
|
49
|
+
description="Names of the streams to try reading from when running a check operation.",
|
50
|
+
examples=[["users"], ["users", "contacts"]],
|
51
|
+
title="Stream Names",
|
53
52
|
)
|
54
53
|
|
55
54
|
|
56
55
|
class ConcurrencyLevel(BaseModel):
|
57
|
-
type: Optional[Literal[
|
56
|
+
type: Optional[Literal["ConcurrencyLevel"]] = None
|
58
57
|
default_concurrency: Union[int, str] = Field(
|
59
58
|
...,
|
60
|
-
description=
|
59
|
+
description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.",
|
61
60
|
examples=[10, "{{ config['num_workers'] or 10 }}"],
|
62
|
-
title=
|
61
|
+
title="Default Concurrency",
|
63
62
|
)
|
64
63
|
max_concurrency: Optional[int] = Field(
|
65
64
|
None,
|
66
|
-
description=
|
65
|
+
description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.",
|
67
66
|
examples=[20, 100],
|
68
|
-
title=
|
67
|
+
title="Max Concurrency",
|
69
68
|
)
|
70
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
69
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
71
70
|
|
72
71
|
|
73
72
|
class ConstantBackoffStrategy(BaseModel):
|
74
|
-
type: Literal[
|
73
|
+
type: Literal["ConstantBackoffStrategy"]
|
75
74
|
backoff_time_in_seconds: Union[float, str] = Field(
|
76
75
|
...,
|
77
|
-
description=
|
76
|
+
description="Backoff time in seconds.",
|
78
77
|
examples=[30, 30.5, "{{ config['backoff_time'] }}"],
|
79
|
-
title=
|
78
|
+
title="Backoff Time",
|
80
79
|
)
|
81
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
80
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
82
81
|
|
83
82
|
|
84
83
|
class CursorPagination(BaseModel):
|
85
|
-
type: Literal[
|
84
|
+
type: Literal["CursorPagination"]
|
86
85
|
cursor_value: str = Field(
|
87
86
|
...,
|
88
|
-
description=
|
87
|
+
description="Value of the cursor defining the next page to fetch.",
|
89
88
|
examples=[
|
90
|
-
|
89
|
+
"{{ headers.link.next.cursor }}",
|
91
90
|
"{{ last_record['key'] }}",
|
92
91
|
"{{ response['nextPage'] }}",
|
93
92
|
],
|
94
|
-
title=
|
93
|
+
title="Cursor Value",
|
95
94
|
)
|
96
95
|
page_size: Optional[int] = Field(
|
97
96
|
None,
|
98
|
-
description=
|
97
|
+
description="The number of records to include in each pages.",
|
99
98
|
examples=[100],
|
100
|
-
title=
|
99
|
+
title="Page Size",
|
101
100
|
)
|
102
101
|
stop_condition: Optional[str] = Field(
|
103
102
|
None,
|
104
|
-
description=
|
103
|
+
description="Template string evaluating when to stop paginating.",
|
105
104
|
examples=[
|
106
|
-
|
105
|
+
"{{ response.data.has_more is false }}",
|
107
106
|
"{{ 'next' not in headers['link'] }}",
|
108
107
|
],
|
109
|
-
title=
|
108
|
+
title="Stop Condition",
|
110
109
|
)
|
111
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
110
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
112
111
|
|
113
112
|
|
114
113
|
class CustomAuthenticator(BaseModel):
|
115
114
|
class Config:
|
116
115
|
extra = Extra.allow
|
117
116
|
|
118
|
-
type: Literal[
|
117
|
+
type: Literal["CustomAuthenticator"]
|
119
118
|
class_name: str = Field(
|
120
119
|
...,
|
121
|
-
description=
|
122
|
-
examples=[
|
123
|
-
title=
|
120
|
+
description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.",
|
121
|
+
examples=["source_railz.components.ShortLivedTokenAuthenticator"],
|
122
|
+
title="Class Name",
|
124
123
|
)
|
125
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
124
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
126
125
|
|
127
126
|
|
128
127
|
class CustomBackoffStrategy(BaseModel):
|
129
128
|
class Config:
|
130
129
|
extra = Extra.allow
|
131
130
|
|
132
|
-
type: Literal[
|
131
|
+
type: Literal["CustomBackoffStrategy"]
|
133
132
|
class_name: str = Field(
|
134
133
|
...,
|
135
|
-
description=
|
136
|
-
examples=[
|
137
|
-
title=
|
134
|
+
description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.",
|
135
|
+
examples=["source_railz.components.MyCustomBackoffStrategy"],
|
136
|
+
title="Class Name",
|
138
137
|
)
|
139
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
138
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
140
139
|
|
141
140
|
|
142
141
|
class CustomErrorHandler(BaseModel):
|
143
142
|
class Config:
|
144
143
|
extra = Extra.allow
|
145
144
|
|
146
|
-
type: Literal[
|
145
|
+
type: Literal["CustomErrorHandler"]
|
147
146
|
class_name: str = Field(
|
148
147
|
...,
|
149
|
-
description=
|
150
|
-
examples=[
|
151
|
-
title=
|
148
|
+
description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.",
|
149
|
+
examples=["source_railz.components.MyCustomErrorHandler"],
|
150
|
+
title="Class Name",
|
152
151
|
)
|
153
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
152
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
154
153
|
|
155
154
|
|
156
155
|
class CustomIncrementalSync(BaseModel):
|
157
156
|
class Config:
|
158
157
|
extra = Extra.allow
|
159
158
|
|
160
|
-
type: Literal[
|
159
|
+
type: Literal["CustomIncrementalSync"]
|
161
160
|
class_name: str = Field(
|
162
161
|
...,
|
163
|
-
description=
|
164
|
-
examples=[
|
165
|
-
title=
|
162
|
+
description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.",
|
163
|
+
examples=["source_railz.components.MyCustomIncrementalSync"],
|
164
|
+
title="Class Name",
|
166
165
|
)
|
167
166
|
cursor_field: str = Field(
|
168
167
|
...,
|
169
|
-
description=
|
168
|
+
description="The location of the value on a record that will be used as a bookmark during sync.",
|
170
169
|
)
|
171
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
170
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
172
171
|
|
173
172
|
|
174
173
|
class CustomPaginationStrategy(BaseModel):
|
175
174
|
class Config:
|
176
175
|
extra = Extra.allow
|
177
176
|
|
178
|
-
type: Literal[
|
177
|
+
type: Literal["CustomPaginationStrategy"]
|
179
178
|
class_name: str = Field(
|
180
179
|
...,
|
181
|
-
description=
|
182
|
-
examples=[
|
183
|
-
title=
|
180
|
+
description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.",
|
181
|
+
examples=["source_railz.components.MyCustomPaginationStrategy"],
|
182
|
+
title="Class Name",
|
184
183
|
)
|
185
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
184
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
186
185
|
|
187
186
|
|
188
187
|
class CustomRecordExtractor(BaseModel):
|
189
188
|
class Config:
|
190
189
|
extra = Extra.allow
|
191
190
|
|
192
|
-
type: Literal[
|
191
|
+
type: Literal["CustomRecordExtractor"]
|
193
192
|
class_name: str = Field(
|
194
193
|
...,
|
195
|
-
description=
|
196
|
-
examples=[
|
197
|
-
title=
|
194
|
+
description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.",
|
195
|
+
examples=["source_railz.components.MyCustomRecordExtractor"],
|
196
|
+
title="Class Name",
|
198
197
|
)
|
199
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
198
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
200
199
|
|
201
200
|
|
202
201
|
class CustomRecordFilter(BaseModel):
|
203
202
|
class Config:
|
204
203
|
extra = Extra.allow
|
205
204
|
|
206
|
-
type: Literal[
|
205
|
+
type: Literal["CustomRecordFilter"]
|
207
206
|
class_name: str = Field(
|
208
207
|
...,
|
209
|
-
description=
|
210
|
-
examples=[
|
211
|
-
title=
|
208
|
+
description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.",
|
209
|
+
examples=["source_railz.components.MyCustomCustomRecordFilter"],
|
210
|
+
title="Class Name",
|
212
211
|
)
|
213
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
212
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
214
213
|
|
215
214
|
|
216
215
|
class CustomRequester(BaseModel):
|
217
216
|
class Config:
|
218
217
|
extra = Extra.allow
|
219
218
|
|
220
|
-
type: Literal[
|
219
|
+
type: Literal["CustomRequester"]
|
221
220
|
class_name: str = Field(
|
222
221
|
...,
|
223
|
-
description=
|
224
|
-
examples=[
|
225
|
-
title=
|
222
|
+
description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.",
|
223
|
+
examples=["source_railz.components.MyCustomRecordExtractor"],
|
224
|
+
title="Class Name",
|
226
225
|
)
|
227
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
226
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
228
227
|
|
229
228
|
|
230
229
|
class CustomRetriever(BaseModel):
|
231
230
|
class Config:
|
232
231
|
extra = Extra.allow
|
233
232
|
|
234
|
-
type: Literal[
|
233
|
+
type: Literal["CustomRetriever"]
|
235
234
|
class_name: str = Field(
|
236
235
|
...,
|
237
|
-
description=
|
238
|
-
examples=[
|
239
|
-
title=
|
236
|
+
description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.",
|
237
|
+
examples=["source_railz.components.MyCustomRetriever"],
|
238
|
+
title="Class Name",
|
240
239
|
)
|
241
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
240
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
242
241
|
|
243
242
|
|
244
243
|
class CustomPartitionRouter(BaseModel):
|
245
244
|
class Config:
|
246
245
|
extra = Extra.allow
|
247
246
|
|
248
|
-
type: Literal[
|
247
|
+
type: Literal["CustomPartitionRouter"]
|
249
248
|
class_name: str = Field(
|
250
249
|
...,
|
251
|
-
description=
|
252
|
-
examples=[
|
253
|
-
title=
|
250
|
+
description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.",
|
251
|
+
examples=["source_railz.components.MyCustomPartitionRouter"],
|
252
|
+
title="Class Name",
|
254
253
|
)
|
255
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
254
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
256
255
|
|
257
256
|
|
258
257
|
class CustomSchemaLoader(BaseModel):
|
259
258
|
class Config:
|
260
259
|
extra = Extra.allow
|
261
260
|
|
262
|
-
type: Literal[
|
261
|
+
type: Literal["CustomSchemaLoader"]
|
263
262
|
class_name: str = Field(
|
264
263
|
...,
|
265
|
-
description=
|
266
|
-
examples=[
|
267
|
-
title=
|
264
|
+
description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.",
|
265
|
+
examples=["source_railz.components.MyCustomSchemaLoader"],
|
266
|
+
title="Class Name",
|
268
267
|
)
|
269
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
268
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
270
269
|
|
271
270
|
|
272
271
|
class CustomStateMigration(BaseModel):
|
273
272
|
class Config:
|
274
273
|
extra = Extra.allow
|
275
274
|
|
276
|
-
type: Literal[
|
275
|
+
type: Literal["CustomStateMigration"]
|
277
276
|
class_name: str = Field(
|
278
277
|
...,
|
279
|
-
description=
|
280
|
-
examples=[
|
281
|
-
title=
|
278
|
+
description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.",
|
279
|
+
examples=["source_railz.components.MyCustomStateMigration"],
|
280
|
+
title="Class Name",
|
282
281
|
)
|
283
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
282
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
284
283
|
|
285
284
|
|
286
285
|
class CustomTransformation(BaseModel):
|
287
286
|
class Config:
|
288
287
|
extra = Extra.allow
|
289
288
|
|
290
|
-
type: Literal[
|
289
|
+
type: Literal["CustomTransformation"]
|
291
290
|
class_name: str = Field(
|
292
291
|
...,
|
293
|
-
description=
|
294
|
-
examples=[
|
295
|
-
title=
|
292
|
+
description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.",
|
293
|
+
examples=["source_railz.components.MyCustomTransformation"],
|
294
|
+
title="Class Name",
|
296
295
|
)
|
297
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
296
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
298
297
|
|
299
298
|
|
300
299
|
class LegacyToPerPartitionStateMigration(BaseModel):
|
301
300
|
class Config:
|
302
301
|
extra = Extra.allow
|
303
302
|
|
304
|
-
type: Optional[Literal[
|
303
|
+
type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None
|
305
304
|
|
306
305
|
|
307
306
|
class Algorithm(Enum):
|
308
|
-
HS256 =
|
309
|
-
HS384 =
|
310
|
-
HS512 =
|
311
|
-
ES256 =
|
312
|
-
ES256K =
|
313
|
-
ES384 =
|
314
|
-
ES512 =
|
315
|
-
RS256 =
|
316
|
-
RS384 =
|
317
|
-
RS512 =
|
318
|
-
PS256 =
|
319
|
-
PS384 =
|
320
|
-
PS512 =
|
321
|
-
EdDSA =
|
307
|
+
HS256 = "HS256"
|
308
|
+
HS384 = "HS384"
|
309
|
+
HS512 = "HS512"
|
310
|
+
ES256 = "ES256"
|
311
|
+
ES256K = "ES256K"
|
312
|
+
ES384 = "ES384"
|
313
|
+
ES512 = "ES512"
|
314
|
+
RS256 = "RS256"
|
315
|
+
RS384 = "RS384"
|
316
|
+
RS512 = "RS512"
|
317
|
+
PS256 = "PS256"
|
318
|
+
PS384 = "PS384"
|
319
|
+
PS512 = "PS512"
|
320
|
+
EdDSA = "EdDSA"
|
322
321
|
|
323
322
|
|
324
323
|
class JwtHeaders(BaseModel):
|
@@ -327,21 +326,21 @@ class JwtHeaders(BaseModel):
|
|
327
326
|
|
328
327
|
kid: Optional[str] = Field(
|
329
328
|
None,
|
330
|
-
description=
|
329
|
+
description="Private key ID for user account.",
|
331
330
|
examples=["{{ config['kid'] }}"],
|
332
|
-
title=
|
331
|
+
title="Key Identifier",
|
333
332
|
)
|
334
333
|
typ: Optional[str] = Field(
|
335
|
-
|
336
|
-
description=
|
337
|
-
examples=[
|
338
|
-
title=
|
334
|
+
"JWT",
|
335
|
+
description="The media type of the complete JWT.",
|
336
|
+
examples=["JWT"],
|
337
|
+
title="Type",
|
339
338
|
)
|
340
339
|
cty: Optional[str] = Field(
|
341
340
|
None,
|
342
|
-
description=
|
343
|
-
examples=[
|
344
|
-
title=
|
341
|
+
description="Content type of JWT header.",
|
342
|
+
examples=["JWT"],
|
343
|
+
title="Content Type",
|
345
344
|
)
|
346
345
|
|
347
346
|
|
@@ -351,28 +350,28 @@ class JwtPayload(BaseModel):
|
|
351
350
|
|
352
351
|
iss: Optional[str] = Field(
|
353
352
|
None,
|
354
|
-
description=
|
353
|
+
description="The user/principal that issued the JWT. Commonly a value unique to the user.",
|
355
354
|
examples=["{{ config['iss'] }}"],
|
356
|
-
title=
|
355
|
+
title="Issuer",
|
357
356
|
)
|
358
357
|
sub: Optional[str] = Field(
|
359
358
|
None,
|
360
|
-
description=
|
361
|
-
title=
|
359
|
+
description="The subject of the JWT. Commonly defined by the API.",
|
360
|
+
title="Subject",
|
362
361
|
)
|
363
362
|
aud: Optional[str] = Field(
|
364
363
|
None,
|
365
|
-
description=
|
366
|
-
examples=[
|
367
|
-
title=
|
364
|
+
description="The recipient that the JWT is intended for. Commonly defined by the API.",
|
365
|
+
examples=["appstoreconnect-v1"],
|
366
|
+
title="Audience",
|
368
367
|
)
|
369
368
|
|
370
369
|
|
371
370
|
class JwtAuthenticator(BaseModel):
|
372
|
-
type: Literal[
|
371
|
+
type: Literal["JwtAuthenticator"]
|
373
372
|
secret_key: str = Field(
|
374
373
|
...,
|
375
|
-
description=
|
374
|
+
description="Secret used to sign the JSON web token.",
|
376
375
|
examples=["{{ config['secret_key'] }}"],
|
377
376
|
)
|
378
377
|
base64_encode_secret_key: Optional[bool] = Field(
|
@@ -381,521 +380,516 @@ class JwtAuthenticator(BaseModel):
|
|
381
380
|
)
|
382
381
|
algorithm: Algorithm = Field(
|
383
382
|
...,
|
384
|
-
description=
|
385
|
-
examples=[
|
383
|
+
description="Algorithm used to sign the JSON web token.",
|
384
|
+
examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"],
|
386
385
|
)
|
387
386
|
token_duration: Optional[int] = Field(
|
388
387
|
1200,
|
389
|
-
description=
|
388
|
+
description="The amount of time in seconds a JWT token can be valid after being issued.",
|
390
389
|
examples=[1200, 3600],
|
391
|
-
title=
|
390
|
+
title="Token Duration",
|
392
391
|
)
|
393
392
|
header_prefix: Optional[str] = Field(
|
394
393
|
None,
|
395
|
-
description=
|
396
|
-
examples=[
|
397
|
-
title=
|
394
|
+
description="The prefix to be used within the Authentication header.",
|
395
|
+
examples=["Bearer", "Basic"],
|
396
|
+
title="Header Prefix",
|
398
397
|
)
|
399
398
|
jwt_headers: Optional[JwtHeaders] = Field(
|
400
399
|
None,
|
401
|
-
description=
|
402
|
-
title=
|
400
|
+
description="JWT headers used when signing JSON web token.",
|
401
|
+
title="JWT Headers",
|
403
402
|
)
|
404
403
|
additional_jwt_headers: Optional[Dict[str, Any]] = Field(
|
405
404
|
None,
|
406
|
-
description=
|
407
|
-
title=
|
405
|
+
description="Additional headers to be included with the JWT headers object.",
|
406
|
+
title="Additional JWT Headers",
|
408
407
|
)
|
409
408
|
jwt_payload: Optional[JwtPayload] = Field(
|
410
409
|
None,
|
411
|
-
description=
|
412
|
-
title=
|
410
|
+
description="JWT Payload used when signing JSON web token.",
|
411
|
+
title="JWT Payload",
|
413
412
|
)
|
414
413
|
additional_jwt_payload: Optional[Dict[str, Any]] = Field(
|
415
414
|
None,
|
416
|
-
description=
|
417
|
-
title=
|
415
|
+
description="Additional properties to be added to the JWT payload.",
|
416
|
+
title="Additional JWT Payload Properties",
|
418
417
|
)
|
419
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
418
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
420
419
|
|
421
420
|
|
422
421
|
class RefreshTokenUpdater(BaseModel):
|
423
422
|
refresh_token_name: Optional[str] = Field(
|
424
|
-
|
425
|
-
description=
|
426
|
-
examples=[
|
427
|
-
title=
|
423
|
+
"refresh_token",
|
424
|
+
description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.",
|
425
|
+
examples=["refresh_token"],
|
426
|
+
title="Refresh Token Property Name",
|
428
427
|
)
|
429
428
|
access_token_config_path: Optional[List[str]] = Field(
|
430
|
-
[
|
431
|
-
description=
|
432
|
-
examples=[[
|
433
|
-
title=
|
429
|
+
["credentials", "access_token"],
|
430
|
+
description="Config path to the access token. Make sure the field actually exists in the config.",
|
431
|
+
examples=[["credentials", "access_token"], ["access_token"]],
|
432
|
+
title="Config Path To Access Token",
|
434
433
|
)
|
435
434
|
refresh_token_config_path: Optional[List[str]] = Field(
|
436
|
-
[
|
437
|
-
description=
|
438
|
-
examples=[[
|
439
|
-
title=
|
435
|
+
["credentials", "refresh_token"],
|
436
|
+
description="Config path to the access token. Make sure the field actually exists in the config.",
|
437
|
+
examples=[["credentials", "refresh_token"], ["refresh_token"]],
|
438
|
+
title="Config Path To Refresh Token",
|
440
439
|
)
|
441
440
|
token_expiry_date_config_path: Optional[List[str]] = Field(
|
442
|
-
[
|
443
|
-
description=
|
444
|
-
examples=[[
|
445
|
-
title=
|
441
|
+
["credentials", "token_expiry_date"],
|
442
|
+
description="Config path to the expiry date. Make sure actually exists in the config.",
|
443
|
+
examples=[["credentials", "token_expiry_date"]],
|
444
|
+
title="Config Path To Expiry Date",
|
446
445
|
)
|
447
446
|
refresh_token_error_status_codes: Optional[List[int]] = Field(
|
448
447
|
[],
|
449
|
-
description=
|
448
|
+
description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error",
|
450
449
|
examples=[[400, 500]],
|
451
|
-
title=
|
450
|
+
title="Refresh Token Error Status Codes",
|
452
451
|
)
|
453
452
|
refresh_token_error_key: Optional[str] = Field(
|
454
|
-
|
455
|
-
description=
|
456
|
-
examples=[
|
457
|
-
title=
|
453
|
+
"",
|
454
|
+
description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).",
|
455
|
+
examples=["error"],
|
456
|
+
title="Refresh Token Error Key",
|
458
457
|
)
|
459
458
|
refresh_token_error_values: Optional[List[str]] = Field(
|
460
459
|
[],
|
461
460
|
description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error',
|
462
|
-
examples=[[
|
463
|
-
title=
|
461
|
+
examples=[["invalid_grant", "invalid_permissions"]],
|
462
|
+
title="Refresh Token Error Values",
|
464
463
|
)
|
465
464
|
|
466
465
|
|
467
466
|
class OAuthAuthenticator(BaseModel):
|
468
|
-
type: Literal[
|
467
|
+
type: Literal["OAuthAuthenticator"]
|
469
468
|
client_id: str = Field(
|
470
469
|
...,
|
471
|
-
description=
|
470
|
+
description="The OAuth client ID. Fill it in the user inputs.",
|
472
471
|
examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"],
|
473
|
-
title=
|
472
|
+
title="Client ID",
|
474
473
|
)
|
475
474
|
client_secret: str = Field(
|
476
475
|
...,
|
477
|
-
description=
|
476
|
+
description="The OAuth client secret. Fill it in the user inputs.",
|
478
477
|
examples=[
|
479
478
|
"{{ config['client_secret }}",
|
480
479
|
"{{ config['credentials']['client_secret }}",
|
481
480
|
],
|
482
|
-
title=
|
481
|
+
title="Client Secret",
|
483
482
|
)
|
484
483
|
refresh_token: Optional[str] = Field(
|
485
484
|
None,
|
486
|
-
description=
|
485
|
+
description="Credential artifact used to get a new access token.",
|
487
486
|
examples=[
|
488
487
|
"{{ config['refresh_token'] }}",
|
489
488
|
"{{ config['credentials]['refresh_token'] }}",
|
490
489
|
],
|
491
|
-
title=
|
490
|
+
title="Refresh Token",
|
492
491
|
)
|
493
492
|
token_refresh_endpoint: str = Field(
|
494
493
|
...,
|
495
|
-
description=
|
496
|
-
examples=[
|
497
|
-
title=
|
494
|
+
description="The full URL to call to obtain a new access token.",
|
495
|
+
examples=["https://connect.squareup.com/oauth2/token"],
|
496
|
+
title="Token Refresh Endpoint",
|
498
497
|
)
|
499
498
|
access_token_name: Optional[str] = Field(
|
500
|
-
|
501
|
-
description=
|
502
|
-
examples=[
|
503
|
-
title=
|
499
|
+
"access_token",
|
500
|
+
description="The name of the property which contains the access token in the response from the token refresh endpoint.",
|
501
|
+
examples=["access_token"],
|
502
|
+
title="Access Token Property Name",
|
504
503
|
)
|
505
504
|
expires_in_name: Optional[str] = Field(
|
506
|
-
|
507
|
-
description=
|
508
|
-
examples=[
|
509
|
-
title=
|
505
|
+
"expires_in",
|
506
|
+
description="The name of the property which contains the expiry date in the response from the token refresh endpoint.",
|
507
|
+
examples=["expires_in"],
|
508
|
+
title="Token Expiry Property Name",
|
510
509
|
)
|
511
510
|
grant_type: Optional[str] = Field(
|
512
|
-
|
513
|
-
description=
|
514
|
-
examples=[
|
515
|
-
title=
|
511
|
+
"refresh_token",
|
512
|
+
description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
|
513
|
+
examples=["refresh_token", "client_credentials"],
|
514
|
+
title="Grant Type",
|
516
515
|
)
|
517
516
|
refresh_request_body: Optional[Dict[str, Any]] = Field(
|
518
517
|
None,
|
519
|
-
description=
|
518
|
+
description="Body of the request sent to get a new access token.",
|
520
519
|
examples=[
|
521
520
|
{
|
522
|
-
|
523
|
-
|
524
|
-
|
521
|
+
"applicationId": "{{ config['application_id'] }}",
|
522
|
+
"applicationSecret": "{{ config['application_secret'] }}",
|
523
|
+
"token": "{{ config['token'] }}",
|
525
524
|
}
|
526
525
|
],
|
527
|
-
title=
|
526
|
+
title="Refresh Request Body",
|
528
527
|
)
|
529
528
|
scopes: Optional[List[str]] = Field(
|
530
529
|
None,
|
531
|
-
description=
|
532
|
-
examples=[
|
533
|
-
|
534
|
-
],
|
535
|
-
title='Scopes',
|
530
|
+
description="List of scopes that should be granted to the access token.",
|
531
|
+
examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]],
|
532
|
+
title="Scopes",
|
536
533
|
)
|
537
534
|
token_expiry_date: Optional[str] = Field(
|
538
535
|
None,
|
539
|
-
description=
|
540
|
-
examples=[
|
541
|
-
title=
|
536
|
+
description="The access token expiry date.",
|
537
|
+
examples=["2023-04-06T07:12:10.421833+00:00", 1680842386],
|
538
|
+
title="Token Expiry Date",
|
542
539
|
)
|
543
540
|
token_expiry_date_format: Optional[str] = Field(
|
544
541
|
None,
|
545
|
-
description=
|
546
|
-
examples=[
|
547
|
-
title=
|
542
|
+
description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.",
|
543
|
+
examples=["%Y-%m-%d %H:%M:%S.%f+00:00"],
|
544
|
+
title="Token Expiry Date Format",
|
548
545
|
)
|
549
546
|
refresh_token_updater: Optional[RefreshTokenUpdater] = Field(
|
550
547
|
None,
|
551
|
-
description=
|
552
|
-
title=
|
548
|
+
description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.",
|
549
|
+
title="Token Updater",
|
553
550
|
)
|
554
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
551
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
555
552
|
|
556
553
|
|
557
554
|
class DpathExtractor(BaseModel):
|
558
|
-
type: Literal[
|
555
|
+
type: Literal["DpathExtractor"]
|
559
556
|
field_path: List[str] = Field(
|
560
557
|
...,
|
561
558
|
description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
|
562
559
|
examples=[
|
563
|
-
[
|
564
|
-
[
|
565
|
-
[
|
566
|
-
[
|
560
|
+
["data"],
|
561
|
+
["data", "records"],
|
562
|
+
["data", "{{ parameters.name }}"],
|
563
|
+
["data", "*", "record"],
|
567
564
|
],
|
568
|
-
title=
|
565
|
+
title="Field Path",
|
569
566
|
)
|
570
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
567
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
571
568
|
|
572
569
|
|
573
570
|
class ExponentialBackoffStrategy(BaseModel):
|
574
|
-
type: Literal[
|
571
|
+
type: Literal["ExponentialBackoffStrategy"]
|
575
572
|
factor: Optional[Union[float, str]] = Field(
|
576
573
|
5,
|
577
|
-
description=
|
578
|
-
examples=[5, 5.5,
|
579
|
-
title=
|
574
|
+
description="Multiplicative constant applied on each retry.",
|
575
|
+
examples=[5, 5.5, "10"],
|
576
|
+
title="Factor",
|
580
577
|
)
|
581
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
578
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
582
579
|
|
583
580
|
|
584
581
|
class SessionTokenRequestBearerAuthenticator(BaseModel):
|
585
|
-
type: Literal[
|
582
|
+
type: Literal["Bearer"]
|
586
583
|
|
587
584
|
|
588
585
|
class HttpMethod(Enum):
|
589
|
-
GET =
|
590
|
-
POST =
|
586
|
+
GET = "GET"
|
587
|
+
POST = "POST"
|
591
588
|
|
592
589
|
|
593
590
|
class Action(Enum):
|
594
|
-
SUCCESS =
|
595
|
-
FAIL =
|
596
|
-
RETRY =
|
597
|
-
IGNORE =
|
598
|
-
RATE_LIMITED =
|
591
|
+
SUCCESS = "SUCCESS"
|
592
|
+
FAIL = "FAIL"
|
593
|
+
RETRY = "RETRY"
|
594
|
+
IGNORE = "IGNORE"
|
595
|
+
RATE_LIMITED = "RATE_LIMITED"
|
599
596
|
|
600
597
|
|
601
598
|
class FailureType(Enum):
|
602
|
-
system_error =
|
603
|
-
config_error =
|
604
|
-
transient_error =
|
599
|
+
system_error = "system_error"
|
600
|
+
config_error = "config_error"
|
601
|
+
transient_error = "transient_error"
|
605
602
|
|
606
603
|
|
607
604
|
class HttpResponseFilter(BaseModel):
|
608
|
-
type: Literal[
|
605
|
+
type: Literal["HttpResponseFilter"]
|
609
606
|
action: Optional[Action] = Field(
|
610
607
|
None,
|
611
|
-
description=
|
612
|
-
examples=[
|
613
|
-
title=
|
608
|
+
description="Action to execute if a response matches the filter.",
|
609
|
+
examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"],
|
610
|
+
title="Action",
|
614
611
|
)
|
615
612
|
failure_type: Optional[FailureType] = Field(
|
616
613
|
None,
|
617
|
-
description=
|
618
|
-
examples=[
|
619
|
-
title=
|
614
|
+
description="Failure type of traced exception if a response matches the filter.",
|
615
|
+
examples=["system_error", "config_error", "transient_error"],
|
616
|
+
title="Failure Type",
|
620
617
|
)
|
621
618
|
error_message: Optional[str] = Field(
|
622
619
|
None,
|
623
|
-
description=
|
624
|
-
title=
|
620
|
+
description="Error Message to display if the response matches the filter.",
|
621
|
+
title="Error Message",
|
625
622
|
)
|
626
623
|
error_message_contains: Optional[str] = Field(
|
627
624
|
None,
|
628
|
-
description=
|
629
|
-
example=[
|
630
|
-
title=
|
625
|
+
description="Match the response if its error message contains the substring.",
|
626
|
+
example=["This API operation is not enabled for this site"],
|
627
|
+
title="Error Message Substring",
|
631
628
|
)
|
632
629
|
http_codes: Optional[List[int]] = Field(
|
633
630
|
None,
|
634
|
-
description=
|
631
|
+
description="Match the response if its HTTP code is included in this list.",
|
635
632
|
examples=[[420, 429], [500]],
|
636
|
-
title=
|
633
|
+
title="HTTP Codes",
|
634
|
+
unique_items=True,
|
637
635
|
)
|
638
636
|
predicate: Optional[str] = Field(
|
639
637
|
None,
|
640
|
-
description=
|
638
|
+
description="Match the response if the predicate evaluates to true.",
|
641
639
|
examples=[
|
642
640
|
"{{ 'Too much requests' in response }}",
|
643
641
|
"{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}",
|
644
642
|
],
|
645
|
-
title=
|
643
|
+
title="Predicate",
|
646
644
|
)
|
647
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
645
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
648
646
|
|
649
647
|
|
650
648
|
class InlineSchemaLoader(BaseModel):
|
651
|
-
type: Literal[
|
649
|
+
type: Literal["InlineSchemaLoader"]
|
652
650
|
schema_: Optional[Dict[str, Any]] = Field(
|
653
651
|
None,
|
654
|
-
alias=
|
652
|
+
alias="schema",
|
655
653
|
description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.',
|
656
|
-
title=
|
654
|
+
title="Schema",
|
657
655
|
)
|
658
656
|
|
659
657
|
|
660
658
|
class JsonFileSchemaLoader(BaseModel):
|
661
|
-
type: Literal[
|
659
|
+
type: Literal["JsonFileSchemaLoader"]
|
662
660
|
file_path: Optional[str] = Field(
|
663
661
|
None,
|
664
662
|
description="Path to the JSON file defining the schema. The path is relative to the connector module's root.",
|
665
|
-
example=[
|
666
|
-
title=
|
663
|
+
example=["./schemas/users.json"],
|
664
|
+
title="File Path",
|
667
665
|
)
|
668
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
666
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
669
667
|
|
670
668
|
|
671
669
|
class JsonDecoder(BaseModel):
|
672
|
-
type: Literal[
|
670
|
+
type: Literal["JsonDecoder"]
|
673
671
|
|
674
672
|
|
675
673
|
class JsonlDecoder(BaseModel):
|
676
|
-
type: Literal[
|
674
|
+
type: Literal["JsonlDecoder"]
|
677
675
|
|
678
676
|
|
679
677
|
class KeysToLower(BaseModel):
|
680
|
-
type: Literal[
|
681
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
678
|
+
type: Literal["KeysToLower"]
|
679
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
682
680
|
|
683
681
|
|
684
682
|
class IterableDecoder(BaseModel):
|
685
|
-
type: Literal[
|
683
|
+
type: Literal["IterableDecoder"]
|
686
684
|
|
687
685
|
|
688
686
|
class XmlDecoder(BaseModel):
|
689
|
-
type: Literal[
|
687
|
+
type: Literal["XmlDecoder"]
|
690
688
|
|
691
689
|
|
692
690
|
class MinMaxDatetime(BaseModel):
|
693
|
-
type: Literal[
|
691
|
+
type: Literal["MinMaxDatetime"]
|
694
692
|
datetime: str = Field(
|
695
693
|
...,
|
696
|
-
description=
|
697
|
-
examples=[
|
698
|
-
title=
|
694
|
+
description="Datetime value.",
|
695
|
+
examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"],
|
696
|
+
title="Datetime",
|
699
697
|
)
|
700
698
|
datetime_format: Optional[str] = Field(
|
701
|
-
|
699
|
+
"",
|
702
700
|
description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
|
703
|
-
examples=[
|
704
|
-
title=
|
701
|
+
examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
|
702
|
+
title="Datetime Format",
|
705
703
|
)
|
706
704
|
max_datetime: Optional[str] = Field(
|
707
705
|
None,
|
708
|
-
description=
|
709
|
-
examples=[
|
710
|
-
title=
|
706
|
+
description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.",
|
707
|
+
examples=["2021-01-01T00:00:00Z", "2021-01-01"],
|
708
|
+
title="Max Datetime",
|
711
709
|
)
|
712
710
|
min_datetime: Optional[str] = Field(
|
713
711
|
None,
|
714
|
-
description=
|
715
|
-
examples=[
|
716
|
-
title=
|
712
|
+
description="Floor applied on the datetime value. Must be formatted with the datetime_format field.",
|
713
|
+
examples=["2010-01-01T00:00:00Z", "2010-01-01"],
|
714
|
+
title="Min Datetime",
|
717
715
|
)
|
718
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
716
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
719
717
|
|
720
718
|
|
721
719
|
class NoAuth(BaseModel):
|
722
|
-
type: Literal[
|
723
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
720
|
+
type: Literal["NoAuth"]
|
721
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
724
722
|
|
725
723
|
|
726
724
|
class NoPagination(BaseModel):
|
727
|
-
type: Literal[
|
725
|
+
type: Literal["NoPagination"]
|
728
726
|
|
729
727
|
|
730
728
|
class OAuthConfigSpecification(BaseModel):
|
731
729
|
class Config:
|
732
730
|
extra = Extra.allow
|
733
731
|
|
734
|
-
oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = (
|
735
|
-
|
736
|
-
|
737
|
-
|
738
|
-
|
739
|
-
|
740
|
-
{
|
741
|
-
|
742
|
-
|
743
|
-
|
744
|
-
|
745
|
-
|
746
|
-
|
747
|
-
title='OAuth user input',
|
748
|
-
)
|
732
|
+
oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field(
|
733
|
+
None,
|
734
|
+
description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }",
|
735
|
+
examples=[
|
736
|
+
{"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}},
|
737
|
+
{
|
738
|
+
"app_id": {
|
739
|
+
"type": "string",
|
740
|
+
"path_in_connector_config": ["info", "app_id"],
|
741
|
+
}
|
742
|
+
},
|
743
|
+
],
|
744
|
+
title="OAuth user input",
|
749
745
|
)
|
750
746
|
complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
|
751
747
|
None,
|
752
748
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
|
753
749
|
examples=[
|
754
750
|
{
|
755
|
-
|
756
|
-
|
757
|
-
|
751
|
+
"refresh_token": {
|
752
|
+
"type": "string,",
|
753
|
+
"path_in_connector_config": ["credentials", "refresh_token"],
|
758
754
|
}
|
759
755
|
}
|
760
756
|
],
|
761
|
-
title=
|
757
|
+
title="OAuth output specification",
|
762
758
|
)
|
763
759
|
complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field(
|
764
760
|
None,
|
765
|
-
description=
|
766
|
-
examples=[
|
767
|
-
|
768
|
-
],
|
769
|
-
title='OAuth input specification',
|
761
|
+
description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }",
|
762
|
+
examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}],
|
763
|
+
title="OAuth input specification",
|
770
764
|
)
|
771
765
|
complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field(
|
772
766
|
None,
|
773
767
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }",
|
774
768
|
examples=[
|
775
769
|
{
|
776
|
-
|
777
|
-
|
778
|
-
|
770
|
+
"client_id": {
|
771
|
+
"type": "string,",
|
772
|
+
"path_in_connector_config": ["credentials", "client_id"],
|
779
773
|
},
|
780
|
-
|
781
|
-
|
782
|
-
|
774
|
+
"client_secret": {
|
775
|
+
"type": "string,",
|
776
|
+
"path_in_connector_config": ["credentials", "client_secret"],
|
783
777
|
},
|
784
778
|
}
|
785
779
|
],
|
786
|
-
title=
|
780
|
+
title="OAuth server output specification",
|
787
781
|
)
|
788
782
|
|
789
783
|
|
790
784
|
class OffsetIncrement(BaseModel):
|
791
|
-
type: Literal[
|
785
|
+
type: Literal["OffsetIncrement"]
|
792
786
|
page_size: Optional[Union[int, str]] = Field(
|
793
787
|
None,
|
794
|
-
description=
|
788
|
+
description="The number of records to include in each pages.",
|
795
789
|
examples=[100, "{{ config['page_size'] }}"],
|
796
|
-
title=
|
790
|
+
title="Limit",
|
797
791
|
)
|
798
792
|
inject_on_first_request: Optional[bool] = Field(
|
799
793
|
False,
|
800
|
-
description=
|
801
|
-
title=
|
794
|
+
description="Using the `offset` with value `0` during the first request",
|
795
|
+
title="Inject Offset",
|
802
796
|
)
|
803
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
797
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
804
798
|
|
805
799
|
|
806
800
|
class PageIncrement(BaseModel):
|
807
|
-
type: Literal[
|
801
|
+
type: Literal["PageIncrement"]
|
808
802
|
page_size: Optional[Union[int, str]] = Field(
|
809
803
|
None,
|
810
|
-
description=
|
811
|
-
examples=[100,
|
812
|
-
title=
|
804
|
+
description="The number of records to include in each pages.",
|
805
|
+
examples=[100, "100", "{{ config['page_size'] }}"],
|
806
|
+
title="Page Size",
|
813
807
|
)
|
814
808
|
start_from_page: Optional[int] = Field(
|
815
809
|
0,
|
816
|
-
description=
|
810
|
+
description="Index of the first page to request.",
|
817
811
|
examples=[0, 1],
|
818
|
-
title=
|
812
|
+
title="Start From Page",
|
819
813
|
)
|
820
814
|
inject_on_first_request: Optional[bool] = Field(
|
821
815
|
False,
|
822
|
-
description=
|
823
|
-
title=
|
816
|
+
description="Using the `page number` with value defined by `start_from_page` during the first request",
|
817
|
+
title="Inject Page Number",
|
824
818
|
)
|
825
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
819
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
826
820
|
|
827
821
|
|
828
822
|
class PrimaryKey(BaseModel):
|
829
823
|
__root__: Union[str, List[str], List[List[str]]] = Field(
|
830
824
|
...,
|
831
|
-
description=
|
832
|
-
examples=[
|
833
|
-
title=
|
825
|
+
description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.",
|
826
|
+
examples=["id", ["code", "type"]],
|
827
|
+
title="Primary Key",
|
834
828
|
)
|
835
829
|
|
836
830
|
|
837
831
|
class RecordFilter(BaseModel):
|
838
|
-
type: Literal[
|
832
|
+
type: Literal["RecordFilter"]
|
839
833
|
condition: Optional[str] = Field(
|
840
|
-
|
841
|
-
description=
|
834
|
+
"",
|
835
|
+
description="The predicate to filter a record. Records will be removed if evaluated to False.",
|
842
836
|
examples=[
|
843
837
|
"{{ record['created_at'] >= stream_interval['start_time'] }}",
|
844
838
|
"{{ record.status in ['active', 'expired'] }}",
|
845
839
|
],
|
846
840
|
)
|
847
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
841
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
848
842
|
|
849
843
|
|
850
844
|
class SchemaNormalization(Enum):
|
851
|
-
None_ =
|
852
|
-
Default =
|
845
|
+
None_ = "None"
|
846
|
+
Default = "Default"
|
853
847
|
|
854
848
|
|
855
849
|
class RemoveFields(BaseModel):
|
856
|
-
type: Literal[
|
850
|
+
type: Literal["RemoveFields"]
|
857
851
|
condition: Optional[str] = Field(
|
858
|
-
|
859
|
-
description=
|
852
|
+
"",
|
853
|
+
description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,",
|
860
854
|
examples=[
|
861
855
|
"{{ property|string == '' }}",
|
862
|
-
|
863
|
-
|
856
|
+
"{{ property is integer }}",
|
857
|
+
"{{ property|length > 5 }}",
|
864
858
|
"{{ property == 'some_string_to_match' }}",
|
865
859
|
],
|
866
860
|
)
|
867
861
|
field_pointers: List[List[str]] = Field(
|
868
862
|
...,
|
869
|
-
description=
|
870
|
-
examples=[[
|
871
|
-
title=
|
863
|
+
description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.",
|
864
|
+
examples=[["tags"], [["content", "html"], ["content", "plain_text"]]],
|
865
|
+
title="Field Paths",
|
872
866
|
)
|
873
867
|
|
874
868
|
|
875
869
|
class RequestPath(BaseModel):
|
876
|
-
type: Literal[
|
870
|
+
type: Literal["RequestPath"]
|
877
871
|
|
878
872
|
|
879
873
|
class InjectInto(Enum):
|
880
|
-
request_parameter =
|
881
|
-
header =
|
882
|
-
body_data =
|
883
|
-
body_json =
|
874
|
+
request_parameter = "request_parameter"
|
875
|
+
header = "header"
|
876
|
+
body_data = "body_data"
|
877
|
+
body_json = "body_json"
|
884
878
|
|
885
879
|
|
886
880
|
class RequestOption(BaseModel):
|
887
|
-
type: Literal[
|
881
|
+
type: Literal["RequestOption"]
|
888
882
|
field_name: str = Field(
|
889
883
|
...,
|
890
|
-
description=
|
891
|
-
examples=[
|
892
|
-
title=
|
884
|
+
description="Configures which key should be used in the location that the descriptor is being injected into",
|
885
|
+
examples=["segment_id"],
|
886
|
+
title="Request Option",
|
893
887
|
)
|
894
888
|
inject_into: InjectInto = Field(
|
895
889
|
...,
|
896
|
-
description=
|
897
|
-
examples=[
|
898
|
-
title=
|
890
|
+
description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.",
|
891
|
+
examples=["request_parameter", "header", "body_data", "body_json"],
|
892
|
+
title="Inject Into",
|
899
893
|
)
|
900
894
|
|
901
895
|
|
@@ -907,54 +901,54 @@ class Schemas(BaseModel):
|
|
907
901
|
|
908
902
|
|
909
903
|
class LegacySessionTokenAuthenticator(BaseModel):
|
910
|
-
type: Literal[
|
904
|
+
type: Literal["LegacySessionTokenAuthenticator"]
|
911
905
|
header: str = Field(
|
912
906
|
...,
|
913
|
-
description=
|
914
|
-
examples=[
|
915
|
-
title=
|
907
|
+
description="The name of the session token header that will be injected in the request",
|
908
|
+
examples=["X-Session"],
|
909
|
+
title="Session Request Header",
|
916
910
|
)
|
917
911
|
login_url: str = Field(
|
918
912
|
...,
|
919
|
-
description=
|
920
|
-
examples=[
|
921
|
-
title=
|
913
|
+
description="Path of the login URL (do not include the base URL)",
|
914
|
+
examples=["session"],
|
915
|
+
title="Login Path",
|
922
916
|
)
|
923
917
|
session_token: Optional[str] = Field(
|
924
918
|
None,
|
925
|
-
description=
|
919
|
+
description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair",
|
926
920
|
example=["{{ config['session_token'] }}"],
|
927
|
-
title=
|
921
|
+
title="Session Token",
|
928
922
|
)
|
929
923
|
session_token_response_key: str = Field(
|
930
924
|
...,
|
931
|
-
description=
|
932
|
-
examples=[
|
933
|
-
title=
|
925
|
+
description="Name of the key of the session token to be extracted from the response",
|
926
|
+
examples=["id"],
|
927
|
+
title="Response Token Response Key",
|
934
928
|
)
|
935
929
|
username: Optional[str] = Field(
|
936
930
|
None,
|
937
|
-
description=
|
931
|
+
description="Username used to authenticate and obtain a session token",
|
938
932
|
examples=[" {{ config['username'] }}"],
|
939
|
-
title=
|
933
|
+
title="Username",
|
940
934
|
)
|
941
935
|
password: Optional[str] = Field(
|
942
|
-
|
943
|
-
description=
|
944
|
-
examples=["{{ config['password'] }}",
|
945
|
-
title=
|
936
|
+
"",
|
937
|
+
description="Password used to authenticate and obtain a session token",
|
938
|
+
examples=["{{ config['password'] }}", ""],
|
939
|
+
title="Password",
|
946
940
|
)
|
947
941
|
validate_session_url: str = Field(
|
948
942
|
...,
|
949
|
-
description=
|
950
|
-
examples=[
|
951
|
-
title=
|
943
|
+
description="Path of the URL to use to validate that the session token is valid (do not include the base URL)",
|
944
|
+
examples=["user/current"],
|
945
|
+
title="Validate Session Path",
|
952
946
|
)
|
953
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
947
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
954
948
|
|
955
949
|
|
956
950
|
class AsyncJobStatusMap(BaseModel):
|
957
|
-
type: Optional[Literal[
|
951
|
+
type: Optional[Literal["AsyncJobStatusMap"]] = None
|
958
952
|
running: List[str]
|
959
953
|
completed: List[str]
|
960
954
|
failed: List[str]
|
@@ -962,65 +956,65 @@ class AsyncJobStatusMap(BaseModel):
|
|
962
956
|
|
963
957
|
|
964
958
|
class ValueType(Enum):
|
965
|
-
string =
|
966
|
-
number =
|
967
|
-
integer =
|
968
|
-
boolean =
|
959
|
+
string = "string"
|
960
|
+
number = "number"
|
961
|
+
integer = "integer"
|
962
|
+
boolean = "boolean"
|
969
963
|
|
970
964
|
|
971
965
|
class WaitTimeFromHeader(BaseModel):
|
972
|
-
type: Literal[
|
966
|
+
type: Literal["WaitTimeFromHeader"]
|
973
967
|
header: str = Field(
|
974
968
|
...,
|
975
|
-
description=
|
976
|
-
examples=[
|
977
|
-
title=
|
969
|
+
description="The name of the response header defining how long to wait before retrying.",
|
970
|
+
examples=["Retry-After"],
|
971
|
+
title="Response Header Name",
|
978
972
|
)
|
979
973
|
regex: Optional[str] = Field(
|
980
974
|
None,
|
981
|
-
description=
|
982
|
-
examples=[
|
983
|
-
title=
|
975
|
+
description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
|
976
|
+
examples=["([-+]?\\d+)"],
|
977
|
+
title="Extraction Regex",
|
984
978
|
)
|
985
979
|
max_waiting_time_in_seconds: Optional[float] = Field(
|
986
980
|
None,
|
987
|
-
description=
|
981
|
+
description="Given the value extracted from the header is greater than this value, stop the stream.",
|
988
982
|
examples=[3600],
|
989
|
-
title=
|
983
|
+
title="Max Waiting Time in Seconds",
|
990
984
|
)
|
991
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
985
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
992
986
|
|
993
987
|
|
994
988
|
class WaitUntilTimeFromHeader(BaseModel):
|
995
|
-
type: Literal[
|
989
|
+
type: Literal["WaitUntilTimeFromHeader"]
|
996
990
|
header: str = Field(
|
997
991
|
...,
|
998
|
-
description=
|
999
|
-
examples=[
|
1000
|
-
title=
|
992
|
+
description="The name of the response header defining how long to wait before retrying.",
|
993
|
+
examples=["wait_time"],
|
994
|
+
title="Response Header",
|
1001
995
|
)
|
1002
996
|
min_wait: Optional[Union[float, str]] = Field(
|
1003
997
|
None,
|
1004
|
-
description=
|
1005
|
-
examples=[10,
|
1006
|
-
title=
|
998
|
+
description="Minimum time to wait before retrying.",
|
999
|
+
examples=[10, "60"],
|
1000
|
+
title="Minimum Wait Time",
|
1007
1001
|
)
|
1008
1002
|
regex: Optional[str] = Field(
|
1009
1003
|
None,
|
1010
|
-
description=
|
1011
|
-
examples=[
|
1012
|
-
title=
|
1004
|
+
description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
|
1005
|
+
examples=["([-+]?\\d+)"],
|
1006
|
+
title="Extraction Regex",
|
1013
1007
|
)
|
1014
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1008
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1015
1009
|
|
1016
1010
|
|
1017
1011
|
class AddedFieldDefinition(BaseModel):
|
1018
|
-
type: Literal[
|
1012
|
+
type: Literal["AddedFieldDefinition"]
|
1019
1013
|
path: List[str] = Field(
|
1020
1014
|
...,
|
1021
|
-
description=
|
1022
|
-
examples=[[
|
1023
|
-
title=
|
1015
|
+
description="List of strings defining the path where to add the value on the record.",
|
1016
|
+
examples=[["segment_id"], ["metadata", "segment_id"]],
|
1017
|
+
title="Path",
|
1024
1018
|
)
|
1025
1019
|
value: str = Field(
|
1026
1020
|
...,
|
@@ -1030,167 +1024,167 @@ class AddedFieldDefinition(BaseModel):
|
|
1030
1024
|
"{{ record['MetaData']['LastUpdatedTime'] }}",
|
1031
1025
|
"{{ stream_partition['segment_id'] }}",
|
1032
1026
|
],
|
1033
|
-
title=
|
1027
|
+
title="Value",
|
1034
1028
|
)
|
1035
1029
|
value_type: Optional[ValueType] = Field(
|
1036
1030
|
None,
|
1037
|
-
description=
|
1038
|
-
title=
|
1031
|
+
description="Type of the value. If not specified, the type will be inferred from the value.",
|
1032
|
+
title="Value Type",
|
1039
1033
|
)
|
1040
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1034
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1041
1035
|
|
1042
1036
|
|
1043
1037
|
class AddFields(BaseModel):
|
1044
|
-
type: Literal[
|
1038
|
+
type: Literal["AddFields"]
|
1045
1039
|
fields: List[AddedFieldDefinition] = Field(
|
1046
1040
|
...,
|
1047
|
-
description=
|
1048
|
-
title=
|
1041
|
+
description="List of transformations (path and corresponding value) that will be added to the record.",
|
1042
|
+
title="Fields",
|
1049
1043
|
)
|
1050
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1044
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1051
1045
|
|
1052
1046
|
|
1053
1047
|
class ApiKeyAuthenticator(BaseModel):
|
1054
|
-
type: Literal[
|
1048
|
+
type: Literal["ApiKeyAuthenticator"]
|
1055
1049
|
api_token: Optional[str] = Field(
|
1056
1050
|
None,
|
1057
|
-
description=
|
1051
|
+
description="The API key to inject in the request. Fill it in the user inputs.",
|
1058
1052
|
examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"],
|
1059
|
-
title=
|
1053
|
+
title="API Key",
|
1060
1054
|
)
|
1061
1055
|
header: Optional[str] = Field(
|
1062
1056
|
None,
|
1063
|
-
description=
|
1064
|
-
examples=[
|
1065
|
-
title=
|
1057
|
+
description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.",
|
1058
|
+
examples=["Authorization", "Api-Token", "X-Auth-Token"],
|
1059
|
+
title="Header Name",
|
1066
1060
|
)
|
1067
1061
|
inject_into: Optional[RequestOption] = Field(
|
1068
1062
|
None,
|
1069
|
-
description=
|
1063
|
+
description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.",
|
1070
1064
|
examples=[
|
1071
|
-
{
|
1072
|
-
{
|
1065
|
+
{"inject_into": "header", "field_name": "Authorization"},
|
1066
|
+
{"inject_into": "request_parameter", "field_name": "authKey"},
|
1073
1067
|
],
|
1074
|
-
title=
|
1068
|
+
title="Inject API Key Into Outgoing HTTP Request",
|
1075
1069
|
)
|
1076
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1070
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1077
1071
|
|
1078
1072
|
|
1079
1073
|
class AuthFlow(BaseModel):
|
1080
1074
|
auth_flow_type: Optional[AuthFlowType] = Field(
|
1081
|
-
None, description=
|
1075
|
+
None, description="The type of auth to use", title="Auth flow type"
|
1082
1076
|
)
|
1083
1077
|
predicate_key: Optional[List[str]] = Field(
|
1084
1078
|
None,
|
1085
|
-
description=
|
1086
|
-
examples=[[
|
1087
|
-
title=
|
1079
|
+
description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.",
|
1080
|
+
examples=[["credentials", "auth_type"]],
|
1081
|
+
title="Predicate key",
|
1088
1082
|
)
|
1089
1083
|
predicate_value: Optional[str] = Field(
|
1090
1084
|
None,
|
1091
|
-
description=
|
1092
|
-
examples=[
|
1093
|
-
title=
|
1085
|
+
description="Value of the predicate_key fields for the advanced auth to be applicable.",
|
1086
|
+
examples=["Oauth"],
|
1087
|
+
title="Predicate value",
|
1094
1088
|
)
|
1095
1089
|
oauth_config_specification: Optional[OAuthConfigSpecification] = None
|
1096
1090
|
|
1097
1091
|
|
1098
1092
|
class DatetimeBasedCursor(BaseModel):
|
1099
|
-
type: Literal[
|
1093
|
+
type: Literal["DatetimeBasedCursor"]
|
1100
1094
|
cursor_field: str = Field(
|
1101
1095
|
...,
|
1102
|
-
description=
|
1103
|
-
examples=[
|
1104
|
-
title=
|
1096
|
+
description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.",
|
1097
|
+
examples=["created_at", "{{ config['record_cursor'] }}"],
|
1098
|
+
title="Cursor Field",
|
1105
1099
|
)
|
1106
1100
|
datetime_format: str = Field(
|
1107
1101
|
...,
|
1108
|
-
description=
|
1109
|
-
examples=[
|
1110
|
-
title=
|
1102
|
+
description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
|
1103
|
+
examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"],
|
1104
|
+
title="Outgoing Datetime Format",
|
1111
1105
|
)
|
1112
1106
|
start_datetime: Union[str, MinMaxDatetime] = Field(
|
1113
1107
|
...,
|
1114
|
-
description=
|
1115
|
-
examples=[
|
1116
|
-
title=
|
1108
|
+
description="The datetime that determines the earliest record that should be synced.",
|
1109
|
+
examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"],
|
1110
|
+
title="Start Datetime",
|
1117
1111
|
)
|
1118
1112
|
cursor_datetime_formats: Optional[List[str]] = Field(
|
1119
1113
|
None,
|
1120
|
-
description=
|
1121
|
-
title=
|
1114
|
+
description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.",
|
1115
|
+
title="Cursor Datetime Formats",
|
1122
1116
|
)
|
1123
1117
|
cursor_granularity: Optional[str] = Field(
|
1124
1118
|
None,
|
1125
|
-
description=
|
1126
|
-
examples=[
|
1127
|
-
title=
|
1119
|
+
description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.",
|
1120
|
+
examples=["PT1S"],
|
1121
|
+
title="Cursor Granularity",
|
1128
1122
|
)
|
1129
1123
|
end_datetime: Optional[Union[str, MinMaxDatetime]] = Field(
|
1130
1124
|
None,
|
1131
|
-
description=
|
1132
|
-
examples=[
|
1133
|
-
title=
|
1125
|
+
description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.",
|
1126
|
+
examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"],
|
1127
|
+
title="End Datetime",
|
1134
1128
|
)
|
1135
1129
|
end_time_option: Optional[RequestOption] = Field(
|
1136
1130
|
None,
|
1137
|
-
description=
|
1138
|
-
title=
|
1131
|
+
description="Optionally configures how the end datetime will be sent in requests to the source API.",
|
1132
|
+
title="Inject End Time Into Outgoing HTTP Request",
|
1139
1133
|
)
|
1140
1134
|
is_data_feed: Optional[bool] = Field(
|
1141
1135
|
None,
|
1142
|
-
description=
|
1143
|
-
title=
|
1136
|
+
description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.",
|
1137
|
+
title="Whether the target API is formatted as a data feed",
|
1144
1138
|
)
|
1145
1139
|
is_client_side_incremental: Optional[bool] = Field(
|
1146
1140
|
None,
|
1147
|
-
description=
|
1148
|
-
title=
|
1141
|
+
description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.",
|
1142
|
+
title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)",
|
1149
1143
|
)
|
1150
1144
|
is_compare_strictly: Optional[bool] = Field(
|
1151
1145
|
False,
|
1152
|
-
description=
|
1153
|
-
title=
|
1146
|
+
description="Set to True if the target API does not accept queries where the start time equal the end time.",
|
1147
|
+
title="Whether to skip requests if the start time equals the end time",
|
1154
1148
|
)
|
1155
1149
|
global_substream_cursor: Optional[bool] = Field(
|
1156
1150
|
False,
|
1157
|
-
description=
|
1158
|
-
title=
|
1151
|
+
description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).",
|
1152
|
+
title="Whether to store cursor as one value instead of per partition",
|
1159
1153
|
)
|
1160
1154
|
lookback_window: Optional[str] = Field(
|
1161
1155
|
None,
|
1162
|
-
description=
|
1163
|
-
examples=[
|
1164
|
-
title=
|
1156
|
+
description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.",
|
1157
|
+
examples=["P1D", "P{{ config['lookback_days'] }}D"],
|
1158
|
+
title="Lookback Window",
|
1165
1159
|
)
|
1166
1160
|
partition_field_end: Optional[str] = Field(
|
1167
1161
|
None,
|
1168
|
-
description=
|
1169
|
-
examples=[
|
1170
|
-
title=
|
1162
|
+
description="Name of the partition start time field.",
|
1163
|
+
examples=["ending_time"],
|
1164
|
+
title="Partition Field End",
|
1171
1165
|
)
|
1172
1166
|
partition_field_start: Optional[str] = Field(
|
1173
1167
|
None,
|
1174
|
-
description=
|
1175
|
-
examples=[
|
1176
|
-
title=
|
1168
|
+
description="Name of the partition end time field.",
|
1169
|
+
examples=["starting_time"],
|
1170
|
+
title="Partition Field Start",
|
1177
1171
|
)
|
1178
1172
|
start_time_option: Optional[RequestOption] = Field(
|
1179
1173
|
None,
|
1180
|
-
description=
|
1181
|
-
title=
|
1174
|
+
description="Optionally configures how the start datetime will be sent in requests to the source API.",
|
1175
|
+
title="Inject Start Time Into Outgoing HTTP Request",
|
1182
1176
|
)
|
1183
1177
|
step: Optional[str] = Field(
|
1184
1178
|
None,
|
1185
|
-
description=
|
1186
|
-
examples=[
|
1187
|
-
title=
|
1179
|
+
description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.",
|
1180
|
+
examples=["P1W", "{{ config['step_increment'] }}"],
|
1181
|
+
title="Step",
|
1188
1182
|
)
|
1189
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1183
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1190
1184
|
|
1191
1185
|
|
1192
1186
|
class DefaultErrorHandler(BaseModel):
|
1193
|
-
type: Literal[
|
1187
|
+
type: Literal["DefaultErrorHandler"]
|
1194
1188
|
backoff_strategies: Optional[
|
1195
1189
|
List[
|
1196
1190
|
Union[
|
@@ -1203,124 +1197,124 @@ class DefaultErrorHandler(BaseModel):
|
|
1203
1197
|
]
|
1204
1198
|
] = Field(
|
1205
1199
|
None,
|
1206
|
-
description=
|
1207
|
-
title=
|
1200
|
+
description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.",
|
1201
|
+
title="Backoff Strategies",
|
1208
1202
|
)
|
1209
1203
|
max_retries: Optional[int] = Field(
|
1210
1204
|
5,
|
1211
|
-
description=
|
1205
|
+
description="The maximum number of time to retry a retryable request before giving up and failing.",
|
1212
1206
|
examples=[5, 0, 10],
|
1213
|
-
title=
|
1207
|
+
title="Max Retry Count",
|
1214
1208
|
)
|
1215
1209
|
response_filters: Optional[List[HttpResponseFilter]] = Field(
|
1216
1210
|
None,
|
1217
1211
|
description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.",
|
1218
|
-
title=
|
1212
|
+
title="Response Filters",
|
1219
1213
|
)
|
1220
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1214
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1221
1215
|
|
1222
1216
|
|
1223
1217
|
class DefaultPaginator(BaseModel):
|
1224
|
-
type: Literal[
|
1218
|
+
type: Literal["DefaultPaginator"]
|
1225
1219
|
pagination_strategy: Union[
|
1226
1220
|
CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement
|
1227
1221
|
] = Field(
|
1228
1222
|
...,
|
1229
|
-
description=
|
1230
|
-
title=
|
1223
|
+
description="Strategy defining how records are paginated.",
|
1224
|
+
title="Pagination Strategy",
|
1231
1225
|
)
|
1232
1226
|
page_size_option: Optional[RequestOption] = None
|
1233
1227
|
page_token_option: Optional[Union[RequestOption, RequestPath]] = None
|
1234
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1228
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1235
1229
|
|
1236
1230
|
|
1237
1231
|
class SessionTokenRequestApiKeyAuthenticator(BaseModel):
|
1238
|
-
type: Literal[
|
1232
|
+
type: Literal["ApiKey"]
|
1239
1233
|
inject_into: RequestOption = Field(
|
1240
1234
|
...,
|
1241
|
-
description=
|
1235
|
+
description="Configure how the API Key will be sent in requests to the source API.",
|
1242
1236
|
examples=[
|
1243
|
-
{
|
1244
|
-
{
|
1237
|
+
{"inject_into": "header", "field_name": "Authorization"},
|
1238
|
+
{"inject_into": "request_parameter", "field_name": "authKey"},
|
1245
1239
|
],
|
1246
|
-
title=
|
1240
|
+
title="Inject API Key Into Outgoing HTTP Request",
|
1247
1241
|
)
|
1248
1242
|
|
1249
1243
|
|
1250
1244
|
class ListPartitionRouter(BaseModel):
|
1251
|
-
type: Literal[
|
1245
|
+
type: Literal["ListPartitionRouter"]
|
1252
1246
|
cursor_field: str = Field(
|
1253
1247
|
...,
|
1254
1248
|
description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.',
|
1255
|
-
examples=[
|
1256
|
-
title=
|
1249
|
+
examples=["section", "{{ config['section_key'] }}"],
|
1250
|
+
title="Current Partition Value Identifier",
|
1257
1251
|
)
|
1258
1252
|
values: Union[str, List[str]] = Field(
|
1259
1253
|
...,
|
1260
|
-
description=
|
1261
|
-
examples=[[
|
1262
|
-
title=
|
1254
|
+
description="The list of attributes being iterated over and used as input for the requests made to the source API.",
|
1255
|
+
examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"],
|
1256
|
+
title="Partition Values",
|
1263
1257
|
)
|
1264
1258
|
request_option: Optional[RequestOption] = Field(
|
1265
1259
|
None,
|
1266
|
-
description=
|
1267
|
-
title=
|
1260
|
+
description="A request option describing where the list value should be injected into and under what field name if applicable.",
|
1261
|
+
title="Inject Partition Value Into Outgoing HTTP Request",
|
1268
1262
|
)
|
1269
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1263
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1270
1264
|
|
1271
1265
|
|
1272
1266
|
class RecordSelector(BaseModel):
|
1273
|
-
type: Literal[
|
1267
|
+
type: Literal["RecordSelector"]
|
1274
1268
|
extractor: Union[CustomRecordExtractor, DpathExtractor]
|
1275
1269
|
record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field(
|
1276
1270
|
None,
|
1277
|
-
description=
|
1278
|
-
title=
|
1271
|
+
description="Responsible for filtering records to be emitted by the Source.",
|
1272
|
+
title="Record Filter",
|
1279
1273
|
)
|
1280
1274
|
schema_normalization: Optional[SchemaNormalization] = SchemaNormalization.None_
|
1281
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1275
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1282
1276
|
|
1283
1277
|
|
1284
1278
|
class Spec(BaseModel):
|
1285
|
-
type: Literal[
|
1279
|
+
type: Literal["Spec"]
|
1286
1280
|
connection_specification: Dict[str, Any] = Field(
|
1287
1281
|
...,
|
1288
|
-
description=
|
1289
|
-
title=
|
1282
|
+
description="A connection specification describing how a the connector can be configured.",
|
1283
|
+
title="Connection Specification",
|
1290
1284
|
)
|
1291
1285
|
documentation_url: Optional[str] = Field(
|
1292
1286
|
None,
|
1293
1287
|
description="URL of the connector's documentation page.",
|
1294
|
-
examples=[
|
1295
|
-
title=
|
1288
|
+
examples=["https://docs.airbyte.com/integrations/sources/dremio"],
|
1289
|
+
title="Documentation URL",
|
1296
1290
|
)
|
1297
1291
|
advanced_auth: Optional[AuthFlow] = Field(
|
1298
1292
|
None,
|
1299
|
-
description=
|
1300
|
-
title=
|
1293
|
+
description="Advanced specification for configuring the authentication flow.",
|
1294
|
+
title="Advanced Auth",
|
1301
1295
|
)
|
1302
1296
|
|
1303
1297
|
|
1304
1298
|
class CompositeErrorHandler(BaseModel):
|
1305
|
-
type: Literal[
|
1299
|
+
type: Literal["CompositeErrorHandler"]
|
1306
1300
|
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field(
|
1307
1301
|
...,
|
1308
|
-
description=
|
1309
|
-
title=
|
1302
|
+
description="List of error handlers to iterate on to determine how to handle a failed response.",
|
1303
|
+
title="Error Handlers",
|
1310
1304
|
)
|
1311
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1305
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1312
1306
|
|
1313
1307
|
|
1314
1308
|
class DeclarativeSource(BaseModel):
|
1315
1309
|
class Config:
|
1316
1310
|
extra = Extra.forbid
|
1317
1311
|
|
1318
|
-
type: Literal[
|
1312
|
+
type: Literal["DeclarativeSource"]
|
1319
1313
|
check: CheckStream
|
1320
1314
|
streams: List[DeclarativeStream]
|
1321
1315
|
version: str = Field(
|
1322
1316
|
...,
|
1323
|
-
description=
|
1317
|
+
description="The version of the Airbyte CDK used to build and test the source.",
|
1324
1318
|
)
|
1325
1319
|
schemas: Optional[Schemas] = None
|
1326
1320
|
definitions: Optional[Dict[str, Any]] = None
|
@@ -1328,11 +1322,11 @@ class DeclarativeSource(BaseModel):
|
|
1328
1322
|
concurrency_level: Optional[ConcurrencyLevel] = None
|
1329
1323
|
metadata: Optional[Dict[str, Any]] = Field(
|
1330
1324
|
None,
|
1331
|
-
description=
|
1325
|
+
description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
|
1332
1326
|
)
|
1333
1327
|
description: Optional[str] = Field(
|
1334
1328
|
None,
|
1335
|
-
description=
|
1329
|
+
description="A description of the connector. It will be presented on the Source documentation page.",
|
1336
1330
|
)
|
1337
1331
|
|
1338
1332
|
|
@@ -1340,12 +1334,12 @@ class SelectiveAuthenticator(BaseModel):
|
|
1340
1334
|
class Config:
|
1341
1335
|
extra = Extra.allow
|
1342
1336
|
|
1343
|
-
type: Literal[
|
1337
|
+
type: Literal["SelectiveAuthenticator"]
|
1344
1338
|
authenticator_selection_path: List[str] = Field(
|
1345
1339
|
...,
|
1346
|
-
description=
|
1347
|
-
examples=[[
|
1348
|
-
title=
|
1340
|
+
description="Path of the field in config with selected authenticator name",
|
1341
|
+
examples=[["auth"], ["auth", "type"]],
|
1342
|
+
title="Authenticator Selection Path",
|
1349
1343
|
)
|
1350
1344
|
authenticators: Dict[
|
1351
1345
|
str,
|
@@ -1362,132 +1356,128 @@ class SelectiveAuthenticator(BaseModel):
|
|
1362
1356
|
],
|
1363
1357
|
] = Field(
|
1364
1358
|
...,
|
1365
|
-
description=
|
1359
|
+
description="Authenticators to select from.",
|
1366
1360
|
examples=[
|
1367
1361
|
{
|
1368
|
-
|
1369
|
-
|
1370
|
-
|
1371
|
-
|
1362
|
+
"authenticators": {
|
1363
|
+
"token": "#/definitions/ApiKeyAuthenticator",
|
1364
|
+
"oauth": "#/definitions/OAuthAuthenticator",
|
1365
|
+
"jwt": "#/definitions/JwtAuthenticator",
|
1372
1366
|
}
|
1373
1367
|
}
|
1374
1368
|
],
|
1375
|
-
title=
|
1369
|
+
title="Authenticators",
|
1376
1370
|
)
|
1377
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1371
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1378
1372
|
|
1379
1373
|
|
1380
1374
|
class DeclarativeStream(BaseModel):
|
1381
1375
|
class Config:
|
1382
1376
|
extra = Extra.allow
|
1383
1377
|
|
1384
|
-
type: Literal[
|
1378
|
+
type: Literal["DeclarativeStream"]
|
1385
1379
|
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
|
1386
1380
|
...,
|
1387
|
-
description=
|
1388
|
-
title=
|
1389
|
-
)
|
1390
|
-
incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = (
|
1391
|
-
Field(
|
1392
|
-
None,
|
1393
|
-
description='Component used to fetch data incrementally based on a time field in the data.',
|
1394
|
-
title='Incremental Sync',
|
1395
|
-
)
|
1381
|
+
description="Component used to coordinate how records are extracted across stream slices and request pages.",
|
1382
|
+
title="Retriever",
|
1396
1383
|
)
|
1397
|
-
|
1398
|
-
|
1384
|
+
incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field(
|
1385
|
+
None,
|
1386
|
+
description="Component used to fetch data incrementally based on a time field in the data.",
|
1387
|
+
title="Incremental Sync",
|
1399
1388
|
)
|
1389
|
+
name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name")
|
1400
1390
|
primary_key: Optional[PrimaryKey] = Field(
|
1401
|
-
|
1391
|
+
"", description="The primary key of the stream.", title="Primary Key"
|
1402
1392
|
)
|
1403
|
-
schema_loader: Optional[
|
1404
|
-
|
1405
|
-
|
1406
|
-
|
1407
|
-
|
1408
|
-
|
1393
|
+
schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]] = (
|
1394
|
+
Field(
|
1395
|
+
None,
|
1396
|
+
description="Component used to retrieve the schema for the current stream.",
|
1397
|
+
title="Schema Loader",
|
1398
|
+
)
|
1409
1399
|
)
|
1410
1400
|
transformations: Optional[
|
1411
1401
|
List[Union[AddFields, CustomTransformation, RemoveFields, KeysToLower]]
|
1412
1402
|
] = Field(
|
1413
1403
|
None,
|
1414
|
-
description=
|
1415
|
-
title=
|
1404
|
+
description="A list of transformations to be applied to each output record.",
|
1405
|
+
title="Transformations",
|
1416
1406
|
)
|
1417
1407
|
state_migrations: Optional[
|
1418
1408
|
List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]
|
1419
1409
|
] = Field(
|
1420
1410
|
[],
|
1421
|
-
description=
|
1422
|
-
title=
|
1411
|
+
description="Array of state migrations to be applied on the input state",
|
1412
|
+
title="State Migrations",
|
1423
1413
|
)
|
1424
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1414
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1425
1415
|
|
1426
1416
|
|
1427
1417
|
class SessionTokenAuthenticator(BaseModel):
|
1428
|
-
type: Literal[
|
1418
|
+
type: Literal["SessionTokenAuthenticator"]
|
1429
1419
|
login_requester: HttpRequester = Field(
|
1430
1420
|
...,
|
1431
|
-
description=
|
1421
|
+
description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.",
|
1432
1422
|
examples=[
|
1433
1423
|
{
|
1434
|
-
|
1435
|
-
|
1436
|
-
|
1437
|
-
|
1438
|
-
|
1439
|
-
|
1440
|
-
|
1424
|
+
"type": "HttpRequester",
|
1425
|
+
"url_base": "https://my_api.com",
|
1426
|
+
"path": "/login",
|
1427
|
+
"authenticator": {
|
1428
|
+
"type": "BasicHttpAuthenticator",
|
1429
|
+
"username": "{{ config.username }}",
|
1430
|
+
"password": "{{ config.password }}",
|
1441
1431
|
},
|
1442
1432
|
}
|
1443
1433
|
],
|
1444
|
-
title=
|
1434
|
+
title="Login Requester",
|
1445
1435
|
)
|
1446
1436
|
session_token_path: List[str] = Field(
|
1447
1437
|
...,
|
1448
|
-
description=
|
1449
|
-
examples=[[
|
1450
|
-
title=
|
1438
|
+
description="The path in the response body returned from the login requester to the session token.",
|
1439
|
+
examples=[["access_token"], ["result", "token"]],
|
1440
|
+
title="Session Token Path",
|
1451
1441
|
)
|
1452
1442
|
expiration_duration: Optional[str] = Field(
|
1453
1443
|
None,
|
1454
|
-
description=
|
1455
|
-
examples=[
|
1456
|
-
title=
|
1444
|
+
description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.",
|
1445
|
+
examples=["PT1H", "P1D"],
|
1446
|
+
title="Expiration Duration",
|
1457
1447
|
)
|
1458
1448
|
request_authentication: Union[
|
1459
1449
|
SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator
|
1460
1450
|
] = Field(
|
1461
1451
|
...,
|
1462
|
-
description=
|
1463
|
-
title=
|
1452
|
+
description="Authentication method to use for requests sent to the API, specifying how to inject the session token.",
|
1453
|
+
title="Data Request Authentication",
|
1464
1454
|
)
|
1465
1455
|
decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field(
|
1466
|
-
None, description=
|
1456
|
+
None, description="Component used to decode the response.", title="Decoder"
|
1467
1457
|
)
|
1468
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1458
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1469
1459
|
|
1470
1460
|
|
1471
1461
|
class HttpRequester(BaseModel):
|
1472
|
-
type: Literal[
|
1462
|
+
type: Literal["HttpRequester"]
|
1473
1463
|
url_base: str = Field(
|
1474
1464
|
...,
|
1475
|
-
description=
|
1465
|
+
description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
|
1476
1466
|
examples=[
|
1477
|
-
|
1467
|
+
"https://connect.squareup.com/v2",
|
1478
1468
|
"{{ config['base_url'] or 'https://app.posthog.com'}}/api/",
|
1479
1469
|
],
|
1480
|
-
title=
|
1470
|
+
title="API Base URL",
|
1481
1471
|
)
|
1482
1472
|
path: str = Field(
|
1483
1473
|
...,
|
1484
|
-
description=
|
1474
|
+
description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
|
1485
1475
|
examples=[
|
1486
|
-
|
1476
|
+
"/products",
|
1487
1477
|
"/quotes/{{ stream_partition['id'] }}/quote_line_groups",
|
1488
1478
|
"/trades/{{ config['symbol_id'] }}/history",
|
1489
1479
|
],
|
1490
|
-
title=
|
1480
|
+
title="URL Path",
|
1491
1481
|
)
|
1492
1482
|
authenticator: Optional[
|
1493
1483
|
Union[
|
@@ -1504,111 +1494,111 @@ class HttpRequester(BaseModel):
|
|
1504
1494
|
]
|
1505
1495
|
] = Field(
|
1506
1496
|
None,
|
1507
|
-
description=
|
1508
|
-
title=
|
1497
|
+
description="Authentication method to use for requests sent to the API.",
|
1498
|
+
title="Authenticator",
|
1509
1499
|
)
|
1510
1500
|
error_handler: Optional[
|
1511
1501
|
Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]
|
1512
1502
|
] = Field(
|
1513
1503
|
None,
|
1514
|
-
description=
|
1515
|
-
title=
|
1504
|
+
description="Error handler component that defines how to handle errors.",
|
1505
|
+
title="Error Handler",
|
1516
1506
|
)
|
1517
1507
|
http_method: Optional[HttpMethod] = Field(
|
1518
1508
|
HttpMethod.GET,
|
1519
|
-
description=
|
1520
|
-
examples=[
|
1521
|
-
title=
|
1509
|
+
description="The HTTP method used to fetch data from the source (can be GET or POST).",
|
1510
|
+
examples=["GET", "POST"],
|
1511
|
+
title="HTTP Method",
|
1522
1512
|
)
|
1523
1513
|
request_body_data: Optional[Union[str, Dict[str, str]]] = Field(
|
1524
1514
|
None,
|
1525
|
-
description=
|
1515
|
+
description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.",
|
1526
1516
|
examples=[
|
1527
1517
|
'[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n'
|
1528
1518
|
],
|
1529
|
-
title=
|
1519
|
+
title="Request Body Payload (Non-JSON)",
|
1530
1520
|
)
|
1531
1521
|
request_body_json: Optional[Union[str, Dict[str, Any]]] = Field(
|
1532
1522
|
None,
|
1533
|
-
description=
|
1523
|
+
description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.",
|
1534
1524
|
examples=[
|
1535
|
-
{
|
1536
|
-
{
|
1537
|
-
{
|
1525
|
+
{"sort_order": "ASC", "sort_field": "CREATED_AT"},
|
1526
|
+
{"key": "{{ config['value'] }}"},
|
1527
|
+
{"sort": {"field": "updated_at", "order": "ascending"}},
|
1538
1528
|
],
|
1539
|
-
title=
|
1529
|
+
title="Request Body JSON Payload",
|
1540
1530
|
)
|
1541
1531
|
request_headers: Optional[Union[str, Dict[str, str]]] = Field(
|
1542
1532
|
None,
|
1543
|
-
description=
|
1544
|
-
examples=[{
|
1545
|
-
title=
|
1533
|
+
description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.",
|
1534
|
+
examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
|
1535
|
+
title="Request Headers",
|
1546
1536
|
)
|
1547
1537
|
request_parameters: Optional[Union[str, Dict[str, str]]] = Field(
|
1548
1538
|
None,
|
1549
|
-
description=
|
1539
|
+
description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
|
1550
1540
|
examples=[
|
1551
|
-
{
|
1541
|
+
{"unit": "day"},
|
1552
1542
|
{
|
1553
|
-
|
1543
|
+
"query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
|
1554
1544
|
},
|
1555
|
-
{
|
1556
|
-
{
|
1545
|
+
{"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
|
1546
|
+
{"sort_by[asc]": "updated_at"},
|
1557
1547
|
],
|
1558
|
-
title=
|
1548
|
+
title="Query Parameters",
|
1559
1549
|
)
|
1560
1550
|
use_cache: Optional[bool] = Field(
|
1561
1551
|
False,
|
1562
|
-
description=
|
1563
|
-
title=
|
1552
|
+
description="Enables stream requests caching. This field is automatically set by the CDK.",
|
1553
|
+
title="Use Cache",
|
1564
1554
|
)
|
1565
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1555
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1566
1556
|
|
1567
1557
|
|
1568
1558
|
class ParentStreamConfig(BaseModel):
|
1569
|
-
type: Literal[
|
1559
|
+
type: Literal["ParentStreamConfig"]
|
1570
1560
|
parent_key: str = Field(
|
1571
1561
|
...,
|
1572
|
-
description=
|
1573
|
-
examples=[
|
1574
|
-
title=
|
1562
|
+
description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.",
|
1563
|
+
examples=["id", "{{ config['parent_record_id'] }}"],
|
1564
|
+
title="Parent Key",
|
1575
1565
|
)
|
1576
1566
|
stream: DeclarativeStream = Field(
|
1577
|
-
..., description=
|
1567
|
+
..., description="Reference to the parent stream.", title="Parent Stream"
|
1578
1568
|
)
|
1579
1569
|
partition_field: str = Field(
|
1580
1570
|
...,
|
1581
|
-
description=
|
1582
|
-
examples=[
|
1583
|
-
title=
|
1571
|
+
description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.",
|
1572
|
+
examples=["parent_id", "{{ config['parent_partition_field'] }}"],
|
1573
|
+
title="Current Parent Key Value Identifier",
|
1584
1574
|
)
|
1585
1575
|
request_option: Optional[RequestOption] = Field(
|
1586
1576
|
None,
|
1587
|
-
description=
|
1588
|
-
title=
|
1577
|
+
description="A request option describing where the parent key value should be injected into and under what field name if applicable.",
|
1578
|
+
title="Request Option",
|
1589
1579
|
)
|
1590
1580
|
incremental_dependency: Optional[bool] = Field(
|
1591
1581
|
False,
|
1592
|
-
description=
|
1593
|
-
title=
|
1582
|
+
description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.",
|
1583
|
+
title="Incremental Dependency",
|
1594
1584
|
)
|
1595
1585
|
extra_fields: Optional[List[List[str]]] = Field(
|
1596
1586
|
None,
|
1597
|
-
description=
|
1598
|
-
title=
|
1587
|
+
description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.",
|
1588
|
+
title="Extra Fields",
|
1599
1589
|
)
|
1600
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1590
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1601
1591
|
|
1602
1592
|
|
1603
1593
|
class SimpleRetriever(BaseModel):
|
1604
|
-
type: Literal[
|
1594
|
+
type: Literal["SimpleRetriever"]
|
1605
1595
|
record_selector: RecordSelector = Field(
|
1606
1596
|
...,
|
1607
|
-
description=
|
1597
|
+
description="Component that describes how to extract records from a HTTP response.",
|
1608
1598
|
)
|
1609
1599
|
requester: Union[CustomRequester, HttpRequester] = Field(
|
1610
1600
|
...,
|
1611
|
-
description=
|
1601
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API.",
|
1612
1602
|
)
|
1613
1603
|
paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
|
1614
1604
|
None,
|
@@ -1616,61 +1606,55 @@ class SimpleRetriever(BaseModel):
|
|
1616
1606
|
)
|
1617
1607
|
ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field(
|
1618
1608
|
False,
|
1619
|
-
description=
|
1609
|
+
description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.",
|
1620
1610
|
)
|
1621
1611
|
partition_router: Optional[
|
1622
1612
|
Union[
|
1623
1613
|
CustomPartitionRouter,
|
1624
1614
|
ListPartitionRouter,
|
1625
1615
|
SubstreamPartitionRouter,
|
1626
|
-
List[
|
1627
|
-
Union[
|
1628
|
-
CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
|
1629
|
-
]
|
1630
|
-
],
|
1616
|
+
List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
|
1631
1617
|
]
|
1632
1618
|
] = Field(
|
1633
1619
|
[],
|
1634
|
-
description=
|
1635
|
-
title=
|
1620
|
+
description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
|
1621
|
+
title="Partition Router",
|
1636
1622
|
)
|
1637
|
-
decoder: Optional[Union[JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder]] = (
|
1638
|
-
|
1639
|
-
|
1640
|
-
|
1641
|
-
title='Decoder',
|
1642
|
-
)
|
1623
|
+
decoder: Optional[Union[JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder]] = Field(
|
1624
|
+
None,
|
1625
|
+
description="Component decoding the response so records can be extracted.",
|
1626
|
+
title="Decoder",
|
1643
1627
|
)
|
1644
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1628
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1645
1629
|
|
1646
1630
|
|
1647
1631
|
class AsyncRetriever(BaseModel):
|
1648
|
-
type: Literal[
|
1632
|
+
type: Literal["AsyncRetriever"]
|
1649
1633
|
record_selector: RecordSelector = Field(
|
1650
1634
|
...,
|
1651
|
-
description=
|
1635
|
+
description="Component that describes how to extract records from a HTTP response.",
|
1652
1636
|
)
|
1653
1637
|
status_mapping: AsyncJobStatusMap = Field(
|
1654
|
-
..., description=
|
1638
|
+
..., description="Async Job Status to Airbyte CDK Async Job Status mapping."
|
1655
1639
|
)
|
1656
1640
|
status_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
|
1657
|
-
..., description=
|
1641
|
+
..., description="Responsible for fetching the actual status of the async job."
|
1658
1642
|
)
|
1659
1643
|
urls_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
|
1660
1644
|
...,
|
1661
|
-
description=
|
1645
|
+
description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.",
|
1662
1646
|
)
|
1663
1647
|
creation_requester: Union[CustomRequester, HttpRequester] = Field(
|
1664
1648
|
...,
|
1665
|
-
description=
|
1649
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.",
|
1666
1650
|
)
|
1667
1651
|
polling_requester: Union[CustomRequester, HttpRequester] = Field(
|
1668
1652
|
...,
|
1669
|
-
description=
|
1653
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.",
|
1670
1654
|
)
|
1671
1655
|
download_requester: Union[CustomRequester, HttpRequester] = Field(
|
1672
1656
|
...,
|
1673
|
-
description=
|
1657
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.",
|
1674
1658
|
)
|
1675
1659
|
download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
|
1676
1660
|
None,
|
@@ -1682,42 +1666,36 @@ class AsyncRetriever(BaseModel):
|
|
1682
1666
|
)
|
1683
1667
|
delete_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
|
1684
1668
|
None,
|
1685
|
-
description=
|
1669
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.",
|
1686
1670
|
)
|
1687
1671
|
partition_router: Optional[
|
1688
1672
|
Union[
|
1689
1673
|
CustomPartitionRouter,
|
1690
1674
|
ListPartitionRouter,
|
1691
1675
|
SubstreamPartitionRouter,
|
1692
|
-
List[
|
1693
|
-
Union[
|
1694
|
-
CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
|
1695
|
-
]
|
1696
|
-
],
|
1676
|
+
List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
|
1697
1677
|
]
|
1698
1678
|
] = Field(
|
1699
1679
|
[],
|
1700
|
-
description=
|
1701
|
-
title=
|
1680
|
+
description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
|
1681
|
+
title="Partition Router",
|
1702
1682
|
)
|
1703
|
-
decoder: Optional[Union[JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder]] = (
|
1704
|
-
|
1705
|
-
|
1706
|
-
|
1707
|
-
title='Decoder',
|
1708
|
-
)
|
1683
|
+
decoder: Optional[Union[JsonDecoder, JsonlDecoder, IterableDecoder, XmlDecoder]] = Field(
|
1684
|
+
None,
|
1685
|
+
description="Component decoding the response so records can be extracted.",
|
1686
|
+
title="Decoder",
|
1709
1687
|
)
|
1710
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1688
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1711
1689
|
|
1712
1690
|
|
1713
1691
|
class SubstreamPartitionRouter(BaseModel):
|
1714
|
-
type: Literal[
|
1692
|
+
type: Literal["SubstreamPartitionRouter"]
|
1715
1693
|
parent_stream_configs: List[ParentStreamConfig] = Field(
|
1716
1694
|
...,
|
1717
|
-
description=
|
1718
|
-
title=
|
1695
|
+
description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.",
|
1696
|
+
title="Parent Stream Configs",
|
1719
1697
|
)
|
1720
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1698
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1721
1699
|
|
1722
1700
|
|
1723
1701
|
CompositeErrorHandler.update_forward_refs()
|