airbyte-cdk 6.5.3rc2__py3-none-any.whl → 6.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/__init__.py +17 -2
- airbyte_cdk/config_observation.py +10 -3
- airbyte_cdk/connector.py +19 -9
- airbyte_cdk/connector_builder/connector_builder_handler.py +28 -8
- airbyte_cdk/connector_builder/main.py +26 -6
- airbyte_cdk/connector_builder/message_grouper.py +95 -25
- airbyte_cdk/destinations/destination.py +47 -14
- airbyte_cdk/destinations/vector_db_based/config.py +36 -14
- airbyte_cdk/destinations/vector_db_based/document_processor.py +49 -11
- airbyte_cdk/destinations/vector_db_based/embedder.py +52 -11
- airbyte_cdk/destinations/vector_db_based/test_utils.py +14 -4
- airbyte_cdk/destinations/vector_db_based/utils.py +8 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +15 -4
- airbyte_cdk/entrypoint.py +82 -26
- airbyte_cdk/exception_handler.py +13 -3
- airbyte_cdk/logger.py +10 -2
- airbyte_cdk/models/airbyte_protocol.py +11 -5
- airbyte_cdk/models/airbyte_protocol_serializers.py +9 -3
- airbyte_cdk/models/well_known_types.py +1 -1
- airbyte_cdk/sources/abstract_source.py +63 -17
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +47 -14
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +25 -7
- airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py +27 -6
- airbyte_cdk/sources/concurrent_source/thread_pool_manager.py +9 -3
- airbyte_cdk/sources/connector_state_manager.py +32 -10
- airbyte_cdk/sources/declarative/async_job/job.py +3 -1
- airbyte_cdk/sources/declarative/async_job/job_orchestrator.py +68 -14
- airbyte_cdk/sources/declarative/async_job/job_tracker.py +24 -6
- airbyte_cdk/sources/declarative/async_job/repository.py +3 -1
- airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +27 -7
- airbyte_cdk/sources/declarative/auth/oauth.py +35 -11
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +3 -1
- airbyte_cdk/sources/declarative/auth/token.py +25 -8
- airbyte_cdk/sources/declarative/checks/check_stream.py +12 -4
- airbyte_cdk/sources/declarative/checks/connection_checker.py +3 -1
- airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py +11 -3
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +106 -50
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +20 -6
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +43 -0
- airbyte_cdk/sources/declarative/declarative_source.py +3 -1
- airbyte_cdk/sources/declarative/declarative_stream.py +27 -6
- airbyte_cdk/sources/declarative/decoders/__init__.py +2 -2
- airbyte_cdk/sources/declarative/decoders/decoder.py +3 -1
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +48 -13
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +3 -1
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +6 -2
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +6 -2
- airbyte_cdk/sources/declarative/extractors/record_filter.py +24 -7
- airbyte_cdk/sources/declarative/extractors/record_selector.py +10 -3
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +15 -5
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +96 -31
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +22 -8
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +46 -15
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +19 -5
- airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py +3 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +20 -2
- airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +5 -1
- airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +10 -3
- airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +6 -2
- airbyte_cdk/sources/declarative/interpolation/interpolation.py +7 -1
- airbyte_cdk/sources/declarative/interpolation/jinja.py +6 -2
- airbyte_cdk/sources/declarative/interpolation/macros.py +19 -4
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +106 -24
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +14 -5
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +697 -678
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +13 -4
- airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py +9 -2
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +802 -232
- airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py +29 -7
- airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +25 -7
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +54 -15
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +6 -2
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/header_helper.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +17 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +15 -5
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +3 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +18 -8
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +16 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +51 -14
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +29 -8
- airbyte_cdk/sources/declarative/requesters/http_requester.py +58 -16
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +49 -14
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +3 -1
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +3 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +17 -5
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +24 -7
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +9 -3
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +3 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +6 -2
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +19 -6
- airbyte_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py +3 -1
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_nested_request_input_provider.py +21 -7
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +18 -6
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +27 -8
- airbyte_cdk/sources/declarative/requesters/requester.py +3 -1
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +12 -5
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +105 -24
- airbyte_cdk/sources/declarative/schema/default_schema_loader.py +3 -1
- airbyte_cdk/sources/declarative/spec/spec.py +8 -2
- airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +3 -1
- airbyte_cdk/sources/declarative/transformations/add_fields.py +12 -3
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +6 -2
- airbyte_cdk/sources/declarative/types.py +8 -1
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +3 -1
- airbyte_cdk/sources/embedded/base_integration.py +14 -4
- airbyte_cdk/sources/embedded/catalog.py +16 -4
- airbyte_cdk/sources/embedded/runner.py +19 -3
- airbyte_cdk/sources/embedded/tools.py +3 -1
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +12 -4
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +27 -7
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +12 -6
- airbyte_cdk/sources/file_based/config/csv_format.py +21 -9
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +6 -2
- airbyte_cdk/sources/file_based/config/unstructured_format.py +10 -3
- airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +2 -4
- airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +7 -2
- airbyte_cdk/sources/file_based/exceptions.py +13 -15
- airbyte_cdk/sources/file_based/file_based_source.py +82 -24
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +16 -5
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +58 -17
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +89 -26
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +25 -7
- airbyte_cdk/sources/file_based/file_types/file_transfer.py +8 -2
- airbyte_cdk/sources/file_based/file_types/file_type_parser.py +4 -1
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +20 -6
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +57 -16
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +64 -15
- airbyte_cdk/sources/file_based/schema_helpers.py +33 -10
- airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py +3 -1
- airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py +16 -5
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +33 -10
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +47 -11
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +13 -22
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +53 -17
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +17 -5
- airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py +3 -1
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +26 -9
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +67 -21
- airbyte_cdk/sources/http_logger.py +5 -1
- airbyte_cdk/sources/message/repository.py +18 -4
- airbyte_cdk/sources/source.py +17 -7
- airbyte_cdk/sources/streams/availability_strategy.py +9 -3
- airbyte_cdk/sources/streams/call_rate.py +63 -19
- airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +31 -7
- airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +6 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +77 -22
- airbyte_cdk/sources/streams/concurrent/cursor.py +56 -20
- airbyte_cdk/sources/streams/concurrent/default_stream.py +9 -2
- airbyte_cdk/sources/streams/concurrent/helpers.py +6 -2
- airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py +9 -2
- airbyte_cdk/sources/streams/concurrent/partition_reader.py +4 -1
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +10 -2
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +6 -2
- airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py +25 -10
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +32 -16
- airbyte_cdk/sources/streams/core.py +77 -22
- airbyte_cdk/sources/streams/http/availability_strategy.py +3 -1
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +4 -1
- airbyte_cdk/sources/streams/http/error_handlers/error_handler.py +3 -1
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +16 -5
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +9 -3
- airbyte_cdk/sources/streams/http/exceptions.py +2 -2
- airbyte_cdk/sources/streams/http/http.py +133 -33
- airbyte_cdk/sources/streams/http/http_client.py +91 -29
- airbyte_cdk/sources/streams/http/rate_limiting.py +23 -7
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +19 -6
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +38 -11
- airbyte_cdk/sources/streams/http/requests_native_auth/token.py +13 -3
- airbyte_cdk/sources/types.py +5 -1
- airbyte_cdk/sources/utils/record_helper.py +12 -3
- airbyte_cdk/sources/utils/schema_helpers.py +9 -3
- airbyte_cdk/sources/utils/slice_logger.py +4 -1
- airbyte_cdk/sources/utils/transform.py +24 -9
- airbyte_cdk/sql/exceptions.py +19 -6
- airbyte_cdk/sql/secrets.py +3 -1
- airbyte_cdk/sql/shared/catalog_providers.py +13 -4
- airbyte_cdk/sql/shared/sql_processor.py +44 -14
- airbyte_cdk/test/catalog_builder.py +19 -8
- airbyte_cdk/test/entrypoint_wrapper.py +27 -8
- airbyte_cdk/test/mock_http/mocker.py +41 -11
- airbyte_cdk/test/mock_http/request.py +9 -3
- airbyte_cdk/test/mock_http/response.py +3 -1
- airbyte_cdk/test/mock_http/response_builder.py +29 -7
- airbyte_cdk/test/state_builder.py +10 -2
- airbyte_cdk/test/utils/data.py +6 -2
- airbyte_cdk/test/utils/http_mocking.py +3 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +3 -1
- airbyte_cdk/utils/analytics_message.py +10 -2
- airbyte_cdk/utils/datetime_format_inferrer.py +4 -1
- airbyte_cdk/utils/mapping_helpers.py +3 -1
- airbyte_cdk/utils/message_utils.py +11 -4
- airbyte_cdk/utils/print_buffer.py +6 -1
- airbyte_cdk/utils/schema_inferrer.py +30 -9
- airbyte_cdk/utils/spec_schema_transformations.py +3 -1
- airbyte_cdk/utils/traced_exception.py +35 -9
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.6.0.dist-info}/METADATA +8 -7
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.6.0.dist-info}/RECORD +200 -200
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.6.0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.5.3rc2.dist-info → airbyte_cdk-6.6.0.dist-info}/WHEEL +0 -0
@@ -4,321 +4,320 @@
|
|
4
4
|
from __future__ import annotations
|
5
5
|
|
6
6
|
from enum import Enum
|
7
|
-
from typing import Any, Dict, List, Optional, Union
|
7
|
+
from typing import Any, Dict, List, Literal, Optional, Union
|
8
8
|
|
9
9
|
from pydantic.v1 import BaseModel, Extra, Field
|
10
|
-
from typing_extensions import Literal
|
11
10
|
|
12
11
|
|
13
12
|
class AuthFlowType(Enum):
|
14
|
-
oauth2_0 =
|
15
|
-
oauth1_0 =
|
13
|
+
oauth2_0 = "oauth2.0"
|
14
|
+
oauth1_0 = "oauth1.0"
|
16
15
|
|
17
16
|
|
18
17
|
class BasicHttpAuthenticator(BaseModel):
|
19
|
-
type: Literal[
|
18
|
+
type: Literal["BasicHttpAuthenticator"]
|
20
19
|
username: str = Field(
|
21
20
|
...,
|
22
|
-
description=
|
21
|
+
description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.",
|
23
22
|
examples=["{{ config['username'] }}", "{{ config['api_key'] }}"],
|
24
|
-
title=
|
23
|
+
title="Username",
|
25
24
|
)
|
26
25
|
password: Optional[str] = Field(
|
27
|
-
|
28
|
-
description=
|
29
|
-
examples=["{{ config['password'] }}",
|
30
|
-
title=
|
26
|
+
"",
|
27
|
+
description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.",
|
28
|
+
examples=["{{ config['password'] }}", ""],
|
29
|
+
title="Password",
|
31
30
|
)
|
32
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
31
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
33
32
|
|
34
33
|
|
35
34
|
class BearerAuthenticator(BaseModel):
|
36
|
-
type: Literal[
|
35
|
+
type: Literal["BearerAuthenticator"]
|
37
36
|
api_token: str = Field(
|
38
37
|
...,
|
39
|
-
description=
|
38
|
+
description="Token to inject as request header for authenticating with the API.",
|
40
39
|
examples=["{{ config['api_key'] }}", "{{ config['token'] }}"],
|
41
|
-
title=
|
40
|
+
title="Bearer Token",
|
42
41
|
)
|
43
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
42
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
44
43
|
|
45
44
|
|
46
45
|
class CheckStream(BaseModel):
|
47
|
-
type: Literal[
|
46
|
+
type: Literal["CheckStream"]
|
48
47
|
stream_names: List[str] = Field(
|
49
48
|
...,
|
50
|
-
description=
|
51
|
-
examples=[[
|
52
|
-
title=
|
49
|
+
description="Names of the streams to try reading from when running a check operation.",
|
50
|
+
examples=[["users"], ["users", "contacts"]],
|
51
|
+
title="Stream Names",
|
53
52
|
)
|
54
53
|
|
55
54
|
|
56
55
|
class ConcurrencyLevel(BaseModel):
|
57
|
-
type: Optional[Literal[
|
56
|
+
type: Optional[Literal["ConcurrencyLevel"]] = None
|
58
57
|
default_concurrency: Union[int, str] = Field(
|
59
58
|
...,
|
60
|
-
description=
|
59
|
+
description="The amount of concurrency that will applied during a sync. This value can be hardcoded or user-defined in the config if different users have varying volume thresholds in the target API.",
|
61
60
|
examples=[10, "{{ config['num_workers'] or 10 }}"],
|
62
|
-
title=
|
61
|
+
title="Default Concurrency",
|
63
62
|
)
|
64
63
|
max_concurrency: Optional[int] = Field(
|
65
64
|
None,
|
66
|
-
description=
|
65
|
+
description="The maximum level of concurrency that will be used during a sync. This becomes a required field when the default_concurrency derives from the config, because it serves as a safeguard against a user-defined threshold that is too high.",
|
67
66
|
examples=[20, 100],
|
68
|
-
title=
|
67
|
+
title="Max Concurrency",
|
69
68
|
)
|
70
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
69
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
71
70
|
|
72
71
|
|
73
72
|
class ConstantBackoffStrategy(BaseModel):
|
74
|
-
type: Literal[
|
73
|
+
type: Literal["ConstantBackoffStrategy"]
|
75
74
|
backoff_time_in_seconds: Union[float, str] = Field(
|
76
75
|
...,
|
77
|
-
description=
|
76
|
+
description="Backoff time in seconds.",
|
78
77
|
examples=[30, 30.5, "{{ config['backoff_time'] }}"],
|
79
|
-
title=
|
78
|
+
title="Backoff Time",
|
80
79
|
)
|
81
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
80
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
82
81
|
|
83
82
|
|
84
83
|
class CursorPagination(BaseModel):
|
85
|
-
type: Literal[
|
84
|
+
type: Literal["CursorPagination"]
|
86
85
|
cursor_value: str = Field(
|
87
86
|
...,
|
88
|
-
description=
|
87
|
+
description="Value of the cursor defining the next page to fetch.",
|
89
88
|
examples=[
|
90
|
-
|
89
|
+
"{{ headers.link.next.cursor }}",
|
91
90
|
"{{ last_record['key'] }}",
|
92
91
|
"{{ response['nextPage'] }}",
|
93
92
|
],
|
94
|
-
title=
|
93
|
+
title="Cursor Value",
|
95
94
|
)
|
96
95
|
page_size: Optional[int] = Field(
|
97
96
|
None,
|
98
|
-
description=
|
97
|
+
description="The number of records to include in each pages.",
|
99
98
|
examples=[100],
|
100
|
-
title=
|
99
|
+
title="Page Size",
|
101
100
|
)
|
102
101
|
stop_condition: Optional[str] = Field(
|
103
102
|
None,
|
104
|
-
description=
|
103
|
+
description="Template string evaluating when to stop paginating.",
|
105
104
|
examples=[
|
106
|
-
|
105
|
+
"{{ response.data.has_more is false }}",
|
107
106
|
"{{ 'next' not in headers['link'] }}",
|
108
107
|
],
|
109
|
-
title=
|
108
|
+
title="Stop Condition",
|
110
109
|
)
|
111
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
110
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
112
111
|
|
113
112
|
|
114
113
|
class CustomAuthenticator(BaseModel):
|
115
114
|
class Config:
|
116
115
|
extra = Extra.allow
|
117
116
|
|
118
|
-
type: Literal[
|
117
|
+
type: Literal["CustomAuthenticator"]
|
119
118
|
class_name: str = Field(
|
120
119
|
...,
|
121
|
-
description=
|
122
|
-
examples=[
|
123
|
-
title=
|
120
|
+
description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.",
|
121
|
+
examples=["source_railz.components.ShortLivedTokenAuthenticator"],
|
122
|
+
title="Class Name",
|
124
123
|
)
|
125
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
124
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
126
125
|
|
127
126
|
|
128
127
|
class CustomBackoffStrategy(BaseModel):
|
129
128
|
class Config:
|
130
129
|
extra = Extra.allow
|
131
130
|
|
132
|
-
type: Literal[
|
131
|
+
type: Literal["CustomBackoffStrategy"]
|
133
132
|
class_name: str = Field(
|
134
133
|
...,
|
135
|
-
description=
|
136
|
-
examples=[
|
137
|
-
title=
|
134
|
+
description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.",
|
135
|
+
examples=["source_railz.components.MyCustomBackoffStrategy"],
|
136
|
+
title="Class Name",
|
138
137
|
)
|
139
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
138
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
140
139
|
|
141
140
|
|
142
141
|
class CustomErrorHandler(BaseModel):
|
143
142
|
class Config:
|
144
143
|
extra = Extra.allow
|
145
144
|
|
146
|
-
type: Literal[
|
145
|
+
type: Literal["CustomErrorHandler"]
|
147
146
|
class_name: str = Field(
|
148
147
|
...,
|
149
|
-
description=
|
150
|
-
examples=[
|
151
|
-
title=
|
148
|
+
description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.",
|
149
|
+
examples=["source_railz.components.MyCustomErrorHandler"],
|
150
|
+
title="Class Name",
|
152
151
|
)
|
153
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
152
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
154
153
|
|
155
154
|
|
156
155
|
class CustomIncrementalSync(BaseModel):
|
157
156
|
class Config:
|
158
157
|
extra = Extra.allow
|
159
158
|
|
160
|
-
type: Literal[
|
159
|
+
type: Literal["CustomIncrementalSync"]
|
161
160
|
class_name: str = Field(
|
162
161
|
...,
|
163
|
-
description=
|
164
|
-
examples=[
|
165
|
-
title=
|
162
|
+
description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.",
|
163
|
+
examples=["source_railz.components.MyCustomIncrementalSync"],
|
164
|
+
title="Class Name",
|
166
165
|
)
|
167
166
|
cursor_field: str = Field(
|
168
167
|
...,
|
169
|
-
description=
|
168
|
+
description="The location of the value on a record that will be used as a bookmark during sync.",
|
170
169
|
)
|
171
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
170
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
172
171
|
|
173
172
|
|
174
173
|
class CustomPaginationStrategy(BaseModel):
|
175
174
|
class Config:
|
176
175
|
extra = Extra.allow
|
177
176
|
|
178
|
-
type: Literal[
|
177
|
+
type: Literal["CustomPaginationStrategy"]
|
179
178
|
class_name: str = Field(
|
180
179
|
...,
|
181
|
-
description=
|
182
|
-
examples=[
|
183
|
-
title=
|
180
|
+
description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.",
|
181
|
+
examples=["source_railz.components.MyCustomPaginationStrategy"],
|
182
|
+
title="Class Name",
|
184
183
|
)
|
185
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
184
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
186
185
|
|
187
186
|
|
188
187
|
class CustomRecordExtractor(BaseModel):
|
189
188
|
class Config:
|
190
189
|
extra = Extra.allow
|
191
190
|
|
192
|
-
type: Literal[
|
191
|
+
type: Literal["CustomRecordExtractor"]
|
193
192
|
class_name: str = Field(
|
194
193
|
...,
|
195
|
-
description=
|
196
|
-
examples=[
|
197
|
-
title=
|
194
|
+
description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.",
|
195
|
+
examples=["source_railz.components.MyCustomRecordExtractor"],
|
196
|
+
title="Class Name",
|
198
197
|
)
|
199
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
198
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
200
199
|
|
201
200
|
|
202
201
|
class CustomRecordFilter(BaseModel):
|
203
202
|
class Config:
|
204
203
|
extra = Extra.allow
|
205
204
|
|
206
|
-
type: Literal[
|
205
|
+
type: Literal["CustomRecordFilter"]
|
207
206
|
class_name: str = Field(
|
208
207
|
...,
|
209
|
-
description=
|
210
|
-
examples=[
|
211
|
-
title=
|
208
|
+
description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.",
|
209
|
+
examples=["source_railz.components.MyCustomCustomRecordFilter"],
|
210
|
+
title="Class Name",
|
212
211
|
)
|
213
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
212
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
214
213
|
|
215
214
|
|
216
215
|
class CustomRequester(BaseModel):
|
217
216
|
class Config:
|
218
217
|
extra = Extra.allow
|
219
218
|
|
220
|
-
type: Literal[
|
219
|
+
type: Literal["CustomRequester"]
|
221
220
|
class_name: str = Field(
|
222
221
|
...,
|
223
|
-
description=
|
224
|
-
examples=[
|
225
|
-
title=
|
222
|
+
description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.",
|
223
|
+
examples=["source_railz.components.MyCustomRecordExtractor"],
|
224
|
+
title="Class Name",
|
226
225
|
)
|
227
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
226
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
228
227
|
|
229
228
|
|
230
229
|
class CustomRetriever(BaseModel):
|
231
230
|
class Config:
|
232
231
|
extra = Extra.allow
|
233
232
|
|
234
|
-
type: Literal[
|
233
|
+
type: Literal["CustomRetriever"]
|
235
234
|
class_name: str = Field(
|
236
235
|
...,
|
237
|
-
description=
|
238
|
-
examples=[
|
239
|
-
title=
|
236
|
+
description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.",
|
237
|
+
examples=["source_railz.components.MyCustomRetriever"],
|
238
|
+
title="Class Name",
|
240
239
|
)
|
241
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
240
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
242
241
|
|
243
242
|
|
244
243
|
class CustomPartitionRouter(BaseModel):
|
245
244
|
class Config:
|
246
245
|
extra = Extra.allow
|
247
246
|
|
248
|
-
type: Literal[
|
247
|
+
type: Literal["CustomPartitionRouter"]
|
249
248
|
class_name: str = Field(
|
250
249
|
...,
|
251
|
-
description=
|
252
|
-
examples=[
|
253
|
-
title=
|
250
|
+
description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.",
|
251
|
+
examples=["source_railz.components.MyCustomPartitionRouter"],
|
252
|
+
title="Class Name",
|
254
253
|
)
|
255
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
254
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
256
255
|
|
257
256
|
|
258
257
|
class CustomSchemaLoader(BaseModel):
|
259
258
|
class Config:
|
260
259
|
extra = Extra.allow
|
261
260
|
|
262
|
-
type: Literal[
|
261
|
+
type: Literal["CustomSchemaLoader"]
|
263
262
|
class_name: str = Field(
|
264
263
|
...,
|
265
|
-
description=
|
266
|
-
examples=[
|
267
|
-
title=
|
264
|
+
description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.",
|
265
|
+
examples=["source_railz.components.MyCustomSchemaLoader"],
|
266
|
+
title="Class Name",
|
268
267
|
)
|
269
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
268
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
270
269
|
|
271
270
|
|
272
271
|
class CustomStateMigration(BaseModel):
|
273
272
|
class Config:
|
274
273
|
extra = Extra.allow
|
275
274
|
|
276
|
-
type: Literal[
|
275
|
+
type: Literal["CustomStateMigration"]
|
277
276
|
class_name: str = Field(
|
278
277
|
...,
|
279
|
-
description=
|
280
|
-
examples=[
|
281
|
-
title=
|
278
|
+
description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_<name>.<package>.<class_name>`.",
|
279
|
+
examples=["source_railz.components.MyCustomStateMigration"],
|
280
|
+
title="Class Name",
|
282
281
|
)
|
283
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
282
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
284
283
|
|
285
284
|
|
286
285
|
class CustomTransformation(BaseModel):
|
287
286
|
class Config:
|
288
287
|
extra = Extra.allow
|
289
288
|
|
290
|
-
type: Literal[
|
289
|
+
type: Literal["CustomTransformation"]
|
291
290
|
class_name: str = Field(
|
292
291
|
...,
|
293
|
-
description=
|
294
|
-
examples=[
|
295
|
-
title=
|
292
|
+
description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.",
|
293
|
+
examples=["source_railz.components.MyCustomTransformation"],
|
294
|
+
title="Class Name",
|
296
295
|
)
|
297
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
296
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
298
297
|
|
299
298
|
|
300
299
|
class LegacyToPerPartitionStateMigration(BaseModel):
|
301
300
|
class Config:
|
302
301
|
extra = Extra.allow
|
303
302
|
|
304
|
-
type: Optional[Literal[
|
303
|
+
type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None
|
305
304
|
|
306
305
|
|
307
306
|
class Algorithm(Enum):
|
308
|
-
HS256 =
|
309
|
-
HS384 =
|
310
|
-
HS512 =
|
311
|
-
ES256 =
|
312
|
-
ES256K =
|
313
|
-
ES384 =
|
314
|
-
ES512 =
|
315
|
-
RS256 =
|
316
|
-
RS384 =
|
317
|
-
RS512 =
|
318
|
-
PS256 =
|
319
|
-
PS384 =
|
320
|
-
PS512 =
|
321
|
-
EdDSA =
|
307
|
+
HS256 = "HS256"
|
308
|
+
HS384 = "HS384"
|
309
|
+
HS512 = "HS512"
|
310
|
+
ES256 = "ES256"
|
311
|
+
ES256K = "ES256K"
|
312
|
+
ES384 = "ES384"
|
313
|
+
ES512 = "ES512"
|
314
|
+
RS256 = "RS256"
|
315
|
+
RS384 = "RS384"
|
316
|
+
RS512 = "RS512"
|
317
|
+
PS256 = "PS256"
|
318
|
+
PS384 = "PS384"
|
319
|
+
PS512 = "PS512"
|
320
|
+
EdDSA = "EdDSA"
|
322
321
|
|
323
322
|
|
324
323
|
class JwtHeaders(BaseModel):
|
@@ -327,21 +326,21 @@ class JwtHeaders(BaseModel):
|
|
327
326
|
|
328
327
|
kid: Optional[str] = Field(
|
329
328
|
None,
|
330
|
-
description=
|
329
|
+
description="Private key ID for user account.",
|
331
330
|
examples=["{{ config['kid'] }}"],
|
332
|
-
title=
|
331
|
+
title="Key Identifier",
|
333
332
|
)
|
334
333
|
typ: Optional[str] = Field(
|
335
|
-
|
336
|
-
description=
|
337
|
-
examples=[
|
338
|
-
title=
|
334
|
+
"JWT",
|
335
|
+
description="The media type of the complete JWT.",
|
336
|
+
examples=["JWT"],
|
337
|
+
title="Type",
|
339
338
|
)
|
340
339
|
cty: Optional[str] = Field(
|
341
340
|
None,
|
342
|
-
description=
|
343
|
-
examples=[
|
344
|
-
title=
|
341
|
+
description="Content type of JWT header.",
|
342
|
+
examples=["JWT"],
|
343
|
+
title="Content Type",
|
345
344
|
)
|
346
345
|
|
347
346
|
|
@@ -351,28 +350,28 @@ class JwtPayload(BaseModel):
|
|
351
350
|
|
352
351
|
iss: Optional[str] = Field(
|
353
352
|
None,
|
354
|
-
description=
|
353
|
+
description="The user/principal that issued the JWT. Commonly a value unique to the user.",
|
355
354
|
examples=["{{ config['iss'] }}"],
|
356
|
-
title=
|
355
|
+
title="Issuer",
|
357
356
|
)
|
358
357
|
sub: Optional[str] = Field(
|
359
358
|
None,
|
360
|
-
description=
|
361
|
-
title=
|
359
|
+
description="The subject of the JWT. Commonly defined by the API.",
|
360
|
+
title="Subject",
|
362
361
|
)
|
363
362
|
aud: Optional[str] = Field(
|
364
363
|
None,
|
365
|
-
description=
|
366
|
-
examples=[
|
367
|
-
title=
|
364
|
+
description="The recipient that the JWT is intended for. Commonly defined by the API.",
|
365
|
+
examples=["appstoreconnect-v1"],
|
366
|
+
title="Audience",
|
368
367
|
)
|
369
368
|
|
370
369
|
|
371
370
|
class JwtAuthenticator(BaseModel):
|
372
|
-
type: Literal[
|
371
|
+
type: Literal["JwtAuthenticator"]
|
373
372
|
secret_key: str = Field(
|
374
373
|
...,
|
375
|
-
description=
|
374
|
+
description="Secret used to sign the JSON web token.",
|
376
375
|
examples=["{{ config['secret_key'] }}"],
|
377
376
|
)
|
378
377
|
base64_encode_secret_key: Optional[bool] = Field(
|
@@ -381,521 +380,539 @@ class JwtAuthenticator(BaseModel):
|
|
381
380
|
)
|
382
381
|
algorithm: Algorithm = Field(
|
383
382
|
...,
|
384
|
-
description=
|
385
|
-
examples=[
|
383
|
+
description="Algorithm used to sign the JSON web token.",
|
384
|
+
examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"],
|
386
385
|
)
|
387
386
|
token_duration: Optional[int] = Field(
|
388
387
|
1200,
|
389
|
-
description=
|
388
|
+
description="The amount of time in seconds a JWT token can be valid after being issued.",
|
390
389
|
examples=[1200, 3600],
|
391
|
-
title=
|
390
|
+
title="Token Duration",
|
392
391
|
)
|
393
392
|
header_prefix: Optional[str] = Field(
|
394
393
|
None,
|
395
|
-
description=
|
396
|
-
examples=[
|
397
|
-
title=
|
394
|
+
description="The prefix to be used within the Authentication header.",
|
395
|
+
examples=["Bearer", "Basic"],
|
396
|
+
title="Header Prefix",
|
398
397
|
)
|
399
398
|
jwt_headers: Optional[JwtHeaders] = Field(
|
400
399
|
None,
|
401
|
-
description=
|
402
|
-
title=
|
400
|
+
description="JWT headers used when signing JSON web token.",
|
401
|
+
title="JWT Headers",
|
403
402
|
)
|
404
403
|
additional_jwt_headers: Optional[Dict[str, Any]] = Field(
|
405
404
|
None,
|
406
|
-
description=
|
407
|
-
title=
|
405
|
+
description="Additional headers to be included with the JWT headers object.",
|
406
|
+
title="Additional JWT Headers",
|
408
407
|
)
|
409
408
|
jwt_payload: Optional[JwtPayload] = Field(
|
410
409
|
None,
|
411
|
-
description=
|
412
|
-
title=
|
410
|
+
description="JWT Payload used when signing JSON web token.",
|
411
|
+
title="JWT Payload",
|
413
412
|
)
|
414
413
|
additional_jwt_payload: Optional[Dict[str, Any]] = Field(
|
415
414
|
None,
|
416
|
-
description=
|
417
|
-
title=
|
415
|
+
description="Additional properties to be added to the JWT payload.",
|
416
|
+
title="Additional JWT Payload Properties",
|
418
417
|
)
|
419
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
418
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
420
419
|
|
421
420
|
|
422
421
|
class RefreshTokenUpdater(BaseModel):
|
423
422
|
refresh_token_name: Optional[str] = Field(
|
424
|
-
|
425
|
-
description=
|
426
|
-
examples=[
|
427
|
-
title=
|
423
|
+
"refresh_token",
|
424
|
+
description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.",
|
425
|
+
examples=["refresh_token"],
|
426
|
+
title="Refresh Token Property Name",
|
428
427
|
)
|
429
428
|
access_token_config_path: Optional[List[str]] = Field(
|
430
|
-
[
|
431
|
-
description=
|
432
|
-
examples=[[
|
433
|
-
title=
|
429
|
+
["credentials", "access_token"],
|
430
|
+
description="Config path to the access token. Make sure the field actually exists in the config.",
|
431
|
+
examples=[["credentials", "access_token"], ["access_token"]],
|
432
|
+
title="Config Path To Access Token",
|
434
433
|
)
|
435
434
|
refresh_token_config_path: Optional[List[str]] = Field(
|
436
|
-
[
|
437
|
-
description=
|
438
|
-
examples=[[
|
439
|
-
title=
|
435
|
+
["credentials", "refresh_token"],
|
436
|
+
description="Config path to the access token. Make sure the field actually exists in the config.",
|
437
|
+
examples=[["credentials", "refresh_token"], ["refresh_token"]],
|
438
|
+
title="Config Path To Refresh Token",
|
440
439
|
)
|
441
440
|
token_expiry_date_config_path: Optional[List[str]] = Field(
|
442
|
-
[
|
443
|
-
description=
|
444
|
-
examples=[[
|
445
|
-
title=
|
441
|
+
["credentials", "token_expiry_date"],
|
442
|
+
description="Config path to the expiry date. Make sure actually exists in the config.",
|
443
|
+
examples=[["credentials", "token_expiry_date"]],
|
444
|
+
title="Config Path To Expiry Date",
|
446
445
|
)
|
447
446
|
refresh_token_error_status_codes: Optional[List[int]] = Field(
|
448
447
|
[],
|
449
|
-
description=
|
448
|
+
description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error",
|
450
449
|
examples=[[400, 500]],
|
451
|
-
title=
|
450
|
+
title="Refresh Token Error Status Codes",
|
452
451
|
)
|
453
452
|
refresh_token_error_key: Optional[str] = Field(
|
454
|
-
|
455
|
-
description=
|
456
|
-
examples=[
|
457
|
-
title=
|
453
|
+
"",
|
454
|
+
description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).",
|
455
|
+
examples=["error"],
|
456
|
+
title="Refresh Token Error Key",
|
458
457
|
)
|
459
458
|
refresh_token_error_values: Optional[List[str]] = Field(
|
460
459
|
[],
|
461
460
|
description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error',
|
462
|
-
examples=[[
|
463
|
-
title=
|
461
|
+
examples=[["invalid_grant", "invalid_permissions"]],
|
462
|
+
title="Refresh Token Error Values",
|
464
463
|
)
|
465
464
|
|
466
465
|
|
467
466
|
class OAuthAuthenticator(BaseModel):
|
468
|
-
type: Literal[
|
467
|
+
type: Literal["OAuthAuthenticator"]
|
469
468
|
client_id: str = Field(
|
470
469
|
...,
|
471
|
-
description=
|
470
|
+
description="The OAuth client ID. Fill it in the user inputs.",
|
472
471
|
examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"],
|
473
|
-
title=
|
472
|
+
title="Client ID",
|
474
473
|
)
|
475
474
|
client_secret: str = Field(
|
476
475
|
...,
|
477
|
-
description=
|
476
|
+
description="The OAuth client secret. Fill it in the user inputs.",
|
478
477
|
examples=[
|
479
478
|
"{{ config['client_secret }}",
|
480
479
|
"{{ config['credentials']['client_secret }}",
|
481
480
|
],
|
482
|
-
title=
|
481
|
+
title="Client Secret",
|
483
482
|
)
|
484
483
|
refresh_token: Optional[str] = Field(
|
485
484
|
None,
|
486
|
-
description=
|
485
|
+
description="Credential artifact used to get a new access token.",
|
487
486
|
examples=[
|
488
487
|
"{{ config['refresh_token'] }}",
|
489
488
|
"{{ config['credentials]['refresh_token'] }}",
|
490
489
|
],
|
491
|
-
title=
|
490
|
+
title="Refresh Token",
|
492
491
|
)
|
493
492
|
token_refresh_endpoint: str = Field(
|
494
493
|
...,
|
495
|
-
description=
|
496
|
-
examples=[
|
497
|
-
title=
|
494
|
+
description="The full URL to call to obtain a new access token.",
|
495
|
+
examples=["https://connect.squareup.com/oauth2/token"],
|
496
|
+
title="Token Refresh Endpoint",
|
498
497
|
)
|
499
498
|
access_token_name: Optional[str] = Field(
|
500
|
-
|
501
|
-
description=
|
502
|
-
examples=[
|
503
|
-
title=
|
499
|
+
"access_token",
|
500
|
+
description="The name of the property which contains the access token in the response from the token refresh endpoint.",
|
501
|
+
examples=["access_token"],
|
502
|
+
title="Access Token Property Name",
|
504
503
|
)
|
505
504
|
expires_in_name: Optional[str] = Field(
|
506
|
-
|
507
|
-
description=
|
508
|
-
examples=[
|
509
|
-
title=
|
505
|
+
"expires_in",
|
506
|
+
description="The name of the property which contains the expiry date in the response from the token refresh endpoint.",
|
507
|
+
examples=["expires_in"],
|
508
|
+
title="Token Expiry Property Name",
|
510
509
|
)
|
511
510
|
grant_type: Optional[str] = Field(
|
512
|
-
|
513
|
-
description=
|
514
|
-
examples=[
|
515
|
-
title=
|
511
|
+
"refresh_token",
|
512
|
+
description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
|
513
|
+
examples=["refresh_token", "client_credentials"],
|
514
|
+
title="Grant Type",
|
516
515
|
)
|
517
516
|
refresh_request_body: Optional[Dict[str, Any]] = Field(
|
518
517
|
None,
|
519
|
-
description=
|
518
|
+
description="Body of the request sent to get a new access token.",
|
520
519
|
examples=[
|
521
520
|
{
|
522
|
-
|
523
|
-
|
524
|
-
|
521
|
+
"applicationId": "{{ config['application_id'] }}",
|
522
|
+
"applicationSecret": "{{ config['application_secret'] }}",
|
523
|
+
"token": "{{ config['token'] }}",
|
525
524
|
}
|
526
525
|
],
|
527
|
-
title=
|
526
|
+
title="Refresh Request Body",
|
528
527
|
)
|
529
528
|
scopes: Optional[List[str]] = Field(
|
530
529
|
None,
|
531
|
-
description=
|
532
|
-
examples=[
|
533
|
-
|
534
|
-
],
|
535
|
-
title='Scopes',
|
530
|
+
description="List of scopes that should be granted to the access token.",
|
531
|
+
examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]],
|
532
|
+
title="Scopes",
|
536
533
|
)
|
537
534
|
token_expiry_date: Optional[str] = Field(
|
538
535
|
None,
|
539
|
-
description=
|
540
|
-
examples=[
|
541
|
-
title=
|
536
|
+
description="The access token expiry date.",
|
537
|
+
examples=["2023-04-06T07:12:10.421833+00:00", 1680842386],
|
538
|
+
title="Token Expiry Date",
|
542
539
|
)
|
543
540
|
token_expiry_date_format: Optional[str] = Field(
|
544
541
|
None,
|
545
|
-
description=
|
546
|
-
examples=[
|
547
|
-
title=
|
542
|
+
description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.",
|
543
|
+
examples=["%Y-%m-%d %H:%M:%S.%f+00:00"],
|
544
|
+
title="Token Expiry Date Format",
|
548
545
|
)
|
549
546
|
refresh_token_updater: Optional[RefreshTokenUpdater] = Field(
|
550
547
|
None,
|
551
|
-
description=
|
552
|
-
title=
|
548
|
+
description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.",
|
549
|
+
title="Token Updater",
|
553
550
|
)
|
554
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
551
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
555
552
|
|
556
553
|
|
557
554
|
class DpathExtractor(BaseModel):
|
558
|
-
type: Literal[
|
555
|
+
type: Literal["DpathExtractor"]
|
559
556
|
field_path: List[str] = Field(
|
560
557
|
...,
|
561
558
|
description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
|
562
559
|
examples=[
|
563
|
-
[
|
564
|
-
[
|
565
|
-
[
|
566
|
-
[
|
560
|
+
["data"],
|
561
|
+
["data", "records"],
|
562
|
+
["data", "{{ parameters.name }}"],
|
563
|
+
["data", "*", "record"],
|
567
564
|
],
|
568
|
-
title=
|
565
|
+
title="Field Path",
|
569
566
|
)
|
570
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
567
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
571
568
|
|
572
569
|
|
573
570
|
class ExponentialBackoffStrategy(BaseModel):
|
574
|
-
type: Literal[
|
571
|
+
type: Literal["ExponentialBackoffStrategy"]
|
575
572
|
factor: Optional[Union[float, str]] = Field(
|
576
573
|
5,
|
577
|
-
description=
|
578
|
-
examples=[5, 5.5,
|
579
|
-
title=
|
574
|
+
description="Multiplicative constant applied on each retry.",
|
575
|
+
examples=[5, 5.5, "10"],
|
576
|
+
title="Factor",
|
580
577
|
)
|
581
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
578
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
582
579
|
|
583
580
|
|
584
581
|
class SessionTokenRequestBearerAuthenticator(BaseModel):
|
585
|
-
type: Literal[
|
582
|
+
type: Literal["Bearer"]
|
586
583
|
|
587
584
|
|
588
585
|
class HttpMethod(Enum):
|
589
|
-
GET =
|
590
|
-
POST =
|
586
|
+
GET = "GET"
|
587
|
+
POST = "POST"
|
591
588
|
|
592
589
|
|
593
590
|
class Action(Enum):
|
594
|
-
SUCCESS =
|
595
|
-
FAIL =
|
596
|
-
RETRY =
|
597
|
-
IGNORE =
|
598
|
-
RATE_LIMITED =
|
591
|
+
SUCCESS = "SUCCESS"
|
592
|
+
FAIL = "FAIL"
|
593
|
+
RETRY = "RETRY"
|
594
|
+
IGNORE = "IGNORE"
|
595
|
+
RATE_LIMITED = "RATE_LIMITED"
|
599
596
|
|
600
597
|
|
601
598
|
class FailureType(Enum):
|
602
|
-
system_error =
|
603
|
-
config_error =
|
604
|
-
transient_error =
|
599
|
+
system_error = "system_error"
|
600
|
+
config_error = "config_error"
|
601
|
+
transient_error = "transient_error"
|
605
602
|
|
606
603
|
|
607
604
|
class HttpResponseFilter(BaseModel):
|
608
|
-
type: Literal[
|
605
|
+
type: Literal["HttpResponseFilter"]
|
609
606
|
action: Optional[Action] = Field(
|
610
607
|
None,
|
611
|
-
description=
|
612
|
-
examples=[
|
613
|
-
title=
|
608
|
+
description="Action to execute if a response matches the filter.",
|
609
|
+
examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"],
|
610
|
+
title="Action",
|
614
611
|
)
|
615
612
|
failure_type: Optional[FailureType] = Field(
|
616
613
|
None,
|
617
|
-
description=
|
618
|
-
examples=[
|
619
|
-
title=
|
614
|
+
description="Failure type of traced exception if a response matches the filter.",
|
615
|
+
examples=["system_error", "config_error", "transient_error"],
|
616
|
+
title="Failure Type",
|
620
617
|
)
|
621
618
|
error_message: Optional[str] = Field(
|
622
619
|
None,
|
623
|
-
description=
|
624
|
-
title=
|
620
|
+
description="Error Message to display if the response matches the filter.",
|
621
|
+
title="Error Message",
|
625
622
|
)
|
626
623
|
error_message_contains: Optional[str] = Field(
|
627
624
|
None,
|
628
|
-
description=
|
629
|
-
example=[
|
630
|
-
title=
|
625
|
+
description="Match the response if its error message contains the substring.",
|
626
|
+
example=["This API operation is not enabled for this site"],
|
627
|
+
title="Error Message Substring",
|
631
628
|
)
|
632
629
|
http_codes: Optional[List[int]] = Field(
|
633
630
|
None,
|
634
|
-
description=
|
631
|
+
description="Match the response if its HTTP code is included in this list.",
|
635
632
|
examples=[[420, 429], [500]],
|
636
|
-
title=
|
633
|
+
title="HTTP Codes",
|
634
|
+
unique_items=True,
|
637
635
|
)
|
638
636
|
predicate: Optional[str] = Field(
|
639
637
|
None,
|
640
|
-
description=
|
638
|
+
description="Match the response if the predicate evaluates to true.",
|
641
639
|
examples=[
|
642
640
|
"{{ 'Too much requests' in response }}",
|
643
641
|
"{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}",
|
644
642
|
],
|
645
|
-
title=
|
643
|
+
title="Predicate",
|
646
644
|
)
|
647
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
645
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
648
646
|
|
649
647
|
|
650
648
|
class InlineSchemaLoader(BaseModel):
|
651
|
-
type: Literal[
|
649
|
+
type: Literal["InlineSchemaLoader"]
|
652
650
|
schema_: Optional[Dict[str, Any]] = Field(
|
653
651
|
None,
|
654
|
-
alias=
|
652
|
+
alias="schema",
|
655
653
|
description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.',
|
656
|
-
title=
|
654
|
+
title="Schema",
|
657
655
|
)
|
658
656
|
|
659
657
|
|
660
658
|
class JsonFileSchemaLoader(BaseModel):
|
661
|
-
type: Literal[
|
659
|
+
type: Literal["JsonFileSchemaLoader"]
|
662
660
|
file_path: Optional[str] = Field(
|
663
661
|
None,
|
664
662
|
description="Path to the JSON file defining the schema. The path is relative to the connector module's root.",
|
665
|
-
example=[
|
666
|
-
title=
|
663
|
+
example=["./schemas/users.json"],
|
664
|
+
title="File Path",
|
667
665
|
)
|
668
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
666
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
669
667
|
|
670
668
|
|
671
669
|
class JsonDecoder(BaseModel):
|
672
|
-
type: Literal[
|
670
|
+
type: Literal["JsonDecoder"]
|
673
671
|
|
674
672
|
|
675
673
|
class JsonlDecoder(BaseModel):
|
676
|
-
type: Literal[
|
674
|
+
type: Literal["JsonlDecoder"]
|
677
675
|
|
678
676
|
|
679
677
|
class KeysToLower(BaseModel):
|
680
|
-
type: Literal[
|
681
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
678
|
+
type: Literal["KeysToLower"]
|
679
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
682
680
|
|
683
681
|
|
684
682
|
class IterableDecoder(BaseModel):
|
685
|
-
type: Literal[
|
683
|
+
type: Literal["IterableDecoder"]
|
686
684
|
|
687
685
|
|
688
686
|
class XmlDecoder(BaseModel):
|
689
|
-
type: Literal[
|
687
|
+
type: Literal["XmlDecoder"]
|
688
|
+
|
689
|
+
|
690
|
+
class CustomDecoder(BaseModel):
|
691
|
+
class Config:
|
692
|
+
extra = Extra.allow
|
693
|
+
|
694
|
+
type: Literal["CustomDecoder"]
|
695
|
+
class_name: str = Field(
|
696
|
+
...,
|
697
|
+
description="Fully-qualified name of the class that will be implementing the custom decoding. Has to be a sub class of Decoder. The format is `source_<name>.<package>.<class_name>`.",
|
698
|
+
examples=["source_amazon_ads.components.GzipJsonlDecoder"],
|
699
|
+
title="Class Name",
|
700
|
+
)
|
701
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
702
|
+
|
703
|
+
|
704
|
+
class GzipJsonDecoder(BaseModel):
|
705
|
+
class Config:
|
706
|
+
extra = Extra.allow
|
707
|
+
|
708
|
+
type: Literal["GzipJsonDecoder"]
|
709
|
+
encoding: Optional[str] = "utf-8"
|
710
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
690
711
|
|
691
712
|
|
692
713
|
class MinMaxDatetime(BaseModel):
|
693
|
-
type: Literal[
|
714
|
+
type: Literal["MinMaxDatetime"]
|
694
715
|
datetime: str = Field(
|
695
716
|
...,
|
696
|
-
description=
|
697
|
-
examples=[
|
698
|
-
title=
|
717
|
+
description="Datetime value.",
|
718
|
+
examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"],
|
719
|
+
title="Datetime",
|
699
720
|
)
|
700
721
|
datetime_format: Optional[str] = Field(
|
701
|
-
|
722
|
+
"",
|
702
723
|
description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
|
703
|
-
examples=[
|
704
|
-
title=
|
724
|
+
examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
|
725
|
+
title="Datetime Format",
|
705
726
|
)
|
706
727
|
max_datetime: Optional[str] = Field(
|
707
728
|
None,
|
708
|
-
description=
|
709
|
-
examples=[
|
710
|
-
title=
|
729
|
+
description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.",
|
730
|
+
examples=["2021-01-01T00:00:00Z", "2021-01-01"],
|
731
|
+
title="Max Datetime",
|
711
732
|
)
|
712
733
|
min_datetime: Optional[str] = Field(
|
713
734
|
None,
|
714
|
-
description=
|
715
|
-
examples=[
|
716
|
-
title=
|
735
|
+
description="Floor applied on the datetime value. Must be formatted with the datetime_format field.",
|
736
|
+
examples=["2010-01-01T00:00:00Z", "2010-01-01"],
|
737
|
+
title="Min Datetime",
|
717
738
|
)
|
718
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
739
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
719
740
|
|
720
741
|
|
721
742
|
class NoAuth(BaseModel):
|
722
|
-
type: Literal[
|
723
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
743
|
+
type: Literal["NoAuth"]
|
744
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
724
745
|
|
725
746
|
|
726
747
|
class NoPagination(BaseModel):
|
727
|
-
type: Literal[
|
748
|
+
type: Literal["NoPagination"]
|
728
749
|
|
729
750
|
|
730
751
|
class OAuthConfigSpecification(BaseModel):
|
731
752
|
class Config:
|
732
753
|
extra = Extra.allow
|
733
754
|
|
734
|
-
oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = (
|
735
|
-
|
736
|
-
|
737
|
-
|
738
|
-
|
739
|
-
|
740
|
-
{
|
741
|
-
|
742
|
-
|
743
|
-
|
744
|
-
|
745
|
-
|
746
|
-
|
747
|
-
title='OAuth user input',
|
748
|
-
)
|
755
|
+
oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field(
|
756
|
+
None,
|
757
|
+
description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }",
|
758
|
+
examples=[
|
759
|
+
{"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}},
|
760
|
+
{
|
761
|
+
"app_id": {
|
762
|
+
"type": "string",
|
763
|
+
"path_in_connector_config": ["info", "app_id"],
|
764
|
+
}
|
765
|
+
},
|
766
|
+
],
|
767
|
+
title="OAuth user input",
|
749
768
|
)
|
750
769
|
complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
|
751
770
|
None,
|
752
771
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
|
753
772
|
examples=[
|
754
773
|
{
|
755
|
-
|
756
|
-
|
757
|
-
|
774
|
+
"refresh_token": {
|
775
|
+
"type": "string,",
|
776
|
+
"path_in_connector_config": ["credentials", "refresh_token"],
|
758
777
|
}
|
759
778
|
}
|
760
779
|
],
|
761
|
-
title=
|
780
|
+
title="OAuth output specification",
|
762
781
|
)
|
763
782
|
complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field(
|
764
783
|
None,
|
765
|
-
description=
|
766
|
-
examples=[
|
767
|
-
|
768
|
-
],
|
769
|
-
title='OAuth input specification',
|
784
|
+
description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }",
|
785
|
+
examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}],
|
786
|
+
title="OAuth input specification",
|
770
787
|
)
|
771
788
|
complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field(
|
772
789
|
None,
|
773
790
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }",
|
774
791
|
examples=[
|
775
792
|
{
|
776
|
-
|
777
|
-
|
778
|
-
|
793
|
+
"client_id": {
|
794
|
+
"type": "string,",
|
795
|
+
"path_in_connector_config": ["credentials", "client_id"],
|
779
796
|
},
|
780
|
-
|
781
|
-
|
782
|
-
|
797
|
+
"client_secret": {
|
798
|
+
"type": "string,",
|
799
|
+
"path_in_connector_config": ["credentials", "client_secret"],
|
783
800
|
},
|
784
801
|
}
|
785
802
|
],
|
786
|
-
title=
|
803
|
+
title="OAuth server output specification",
|
787
804
|
)
|
788
805
|
|
789
806
|
|
790
807
|
class OffsetIncrement(BaseModel):
|
791
|
-
type: Literal[
|
808
|
+
type: Literal["OffsetIncrement"]
|
792
809
|
page_size: Optional[Union[int, str]] = Field(
|
793
810
|
None,
|
794
|
-
description=
|
811
|
+
description="The number of records to include in each pages.",
|
795
812
|
examples=[100, "{{ config['page_size'] }}"],
|
796
|
-
title=
|
813
|
+
title="Limit",
|
797
814
|
)
|
798
815
|
inject_on_first_request: Optional[bool] = Field(
|
799
816
|
False,
|
800
|
-
description=
|
801
|
-
title=
|
817
|
+
description="Using the `offset` with value `0` during the first request",
|
818
|
+
title="Inject Offset",
|
802
819
|
)
|
803
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
820
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
804
821
|
|
805
822
|
|
806
823
|
class PageIncrement(BaseModel):
|
807
|
-
type: Literal[
|
824
|
+
type: Literal["PageIncrement"]
|
808
825
|
page_size: Optional[Union[int, str]] = Field(
|
809
826
|
None,
|
810
|
-
description=
|
811
|
-
examples=[100,
|
812
|
-
title=
|
827
|
+
description="The number of records to include in each pages.",
|
828
|
+
examples=[100, "100", "{{ config['page_size'] }}"],
|
829
|
+
title="Page Size",
|
813
830
|
)
|
814
831
|
start_from_page: Optional[int] = Field(
|
815
832
|
0,
|
816
|
-
description=
|
833
|
+
description="Index of the first page to request.",
|
817
834
|
examples=[0, 1],
|
818
|
-
title=
|
835
|
+
title="Start From Page",
|
819
836
|
)
|
820
837
|
inject_on_first_request: Optional[bool] = Field(
|
821
838
|
False,
|
822
|
-
description=
|
823
|
-
title=
|
839
|
+
description="Using the `page number` with value defined by `start_from_page` during the first request",
|
840
|
+
title="Inject Page Number",
|
824
841
|
)
|
825
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
842
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
826
843
|
|
827
844
|
|
828
845
|
class PrimaryKey(BaseModel):
|
829
846
|
__root__: Union[str, List[str], List[List[str]]] = Field(
|
830
847
|
...,
|
831
|
-
description=
|
832
|
-
examples=[
|
833
|
-
title=
|
848
|
+
description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.",
|
849
|
+
examples=["id", ["code", "type"]],
|
850
|
+
title="Primary Key",
|
834
851
|
)
|
835
852
|
|
836
853
|
|
837
854
|
class RecordFilter(BaseModel):
|
838
|
-
type: Literal[
|
855
|
+
type: Literal["RecordFilter"]
|
839
856
|
condition: Optional[str] = Field(
|
840
|
-
|
841
|
-
description=
|
857
|
+
"",
|
858
|
+
description="The predicate to filter a record. Records will be removed if evaluated to False.",
|
842
859
|
examples=[
|
843
860
|
"{{ record['created_at'] >= stream_interval['start_time'] }}",
|
844
861
|
"{{ record.status in ['active', 'expired'] }}",
|
845
862
|
],
|
846
863
|
)
|
847
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
864
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
848
865
|
|
849
866
|
|
850
867
|
class SchemaNormalization(Enum):
|
851
|
-
None_ =
|
852
|
-
Default =
|
868
|
+
None_ = "None"
|
869
|
+
Default = "Default"
|
853
870
|
|
854
871
|
|
855
872
|
class RemoveFields(BaseModel):
|
856
|
-
type: Literal[
|
873
|
+
type: Literal["RemoveFields"]
|
857
874
|
condition: Optional[str] = Field(
|
858
|
-
|
859
|
-
description=
|
875
|
+
"",
|
876
|
+
description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,",
|
860
877
|
examples=[
|
861
878
|
"{{ property|string == '' }}",
|
862
|
-
|
863
|
-
|
879
|
+
"{{ property is integer }}",
|
880
|
+
"{{ property|length > 5 }}",
|
864
881
|
"{{ property == 'some_string_to_match' }}",
|
865
882
|
],
|
866
883
|
)
|
867
884
|
field_pointers: List[List[str]] = Field(
|
868
885
|
...,
|
869
|
-
description=
|
870
|
-
examples=[[
|
871
|
-
title=
|
886
|
+
description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.",
|
887
|
+
examples=[["tags"], [["content", "html"], ["content", "plain_text"]]],
|
888
|
+
title="Field Paths",
|
872
889
|
)
|
873
890
|
|
874
891
|
|
875
892
|
class RequestPath(BaseModel):
|
876
|
-
type: Literal[
|
893
|
+
type: Literal["RequestPath"]
|
877
894
|
|
878
895
|
|
879
896
|
class InjectInto(Enum):
|
880
|
-
request_parameter =
|
881
|
-
header =
|
882
|
-
body_data =
|
883
|
-
body_json =
|
897
|
+
request_parameter = "request_parameter"
|
898
|
+
header = "header"
|
899
|
+
body_data = "body_data"
|
900
|
+
body_json = "body_json"
|
884
901
|
|
885
902
|
|
886
903
|
class RequestOption(BaseModel):
|
887
|
-
type: Literal[
|
904
|
+
type: Literal["RequestOption"]
|
888
905
|
field_name: str = Field(
|
889
906
|
...,
|
890
|
-
description=
|
891
|
-
examples=[
|
892
|
-
title=
|
907
|
+
description="Configures which key should be used in the location that the descriptor is being injected into",
|
908
|
+
examples=["segment_id"],
|
909
|
+
title="Request Option",
|
893
910
|
)
|
894
911
|
inject_into: InjectInto = Field(
|
895
912
|
...,
|
896
|
-
description=
|
897
|
-
examples=[
|
898
|
-
title=
|
913
|
+
description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.",
|
914
|
+
examples=["request_parameter", "header", "body_data", "body_json"],
|
915
|
+
title="Inject Into",
|
899
916
|
)
|
900
917
|
|
901
918
|
|
@@ -907,54 +924,54 @@ class Schemas(BaseModel):
|
|
907
924
|
|
908
925
|
|
909
926
|
class LegacySessionTokenAuthenticator(BaseModel):
|
910
|
-
type: Literal[
|
927
|
+
type: Literal["LegacySessionTokenAuthenticator"]
|
911
928
|
header: str = Field(
|
912
929
|
...,
|
913
|
-
description=
|
914
|
-
examples=[
|
915
|
-
title=
|
930
|
+
description="The name of the session token header that will be injected in the request",
|
931
|
+
examples=["X-Session"],
|
932
|
+
title="Session Request Header",
|
916
933
|
)
|
917
934
|
login_url: str = Field(
|
918
935
|
...,
|
919
|
-
description=
|
920
|
-
examples=[
|
921
|
-
title=
|
936
|
+
description="Path of the login URL (do not include the base URL)",
|
937
|
+
examples=["session"],
|
938
|
+
title="Login Path",
|
922
939
|
)
|
923
940
|
session_token: Optional[str] = Field(
|
924
941
|
None,
|
925
|
-
description=
|
942
|
+
description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair",
|
926
943
|
example=["{{ config['session_token'] }}"],
|
927
|
-
title=
|
944
|
+
title="Session Token",
|
928
945
|
)
|
929
946
|
session_token_response_key: str = Field(
|
930
947
|
...,
|
931
|
-
description=
|
932
|
-
examples=[
|
933
|
-
title=
|
948
|
+
description="Name of the key of the session token to be extracted from the response",
|
949
|
+
examples=["id"],
|
950
|
+
title="Response Token Response Key",
|
934
951
|
)
|
935
952
|
username: Optional[str] = Field(
|
936
953
|
None,
|
937
|
-
description=
|
954
|
+
description="Username used to authenticate and obtain a session token",
|
938
955
|
examples=[" {{ config['username'] }}"],
|
939
|
-
title=
|
956
|
+
title="Username",
|
940
957
|
)
|
941
958
|
password: Optional[str] = Field(
|
942
|
-
|
943
|
-
description=
|
944
|
-
examples=["{{ config['password'] }}",
|
945
|
-
title=
|
959
|
+
"",
|
960
|
+
description="Password used to authenticate and obtain a session token",
|
961
|
+
examples=["{{ config['password'] }}", ""],
|
962
|
+
title="Password",
|
946
963
|
)
|
947
964
|
validate_session_url: str = Field(
|
948
965
|
...,
|
949
|
-
description=
|
950
|
-
examples=[
|
951
|
-
title=
|
966
|
+
description="Path of the URL to use to validate that the session token is valid (do not include the base URL)",
|
967
|
+
examples=["user/current"],
|
968
|
+
title="Validate Session Path",
|
952
969
|
)
|
953
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
970
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
954
971
|
|
955
972
|
|
956
973
|
class AsyncJobStatusMap(BaseModel):
|
957
|
-
type: Optional[Literal[
|
974
|
+
type: Optional[Literal["AsyncJobStatusMap"]] = None
|
958
975
|
running: List[str]
|
959
976
|
completed: List[str]
|
960
977
|
failed: List[str]
|
@@ -962,65 +979,65 @@ class AsyncJobStatusMap(BaseModel):
|
|
962
979
|
|
963
980
|
|
964
981
|
class ValueType(Enum):
|
965
|
-
string =
|
966
|
-
number =
|
967
|
-
integer =
|
968
|
-
boolean =
|
982
|
+
string = "string"
|
983
|
+
number = "number"
|
984
|
+
integer = "integer"
|
985
|
+
boolean = "boolean"
|
969
986
|
|
970
987
|
|
971
988
|
class WaitTimeFromHeader(BaseModel):
|
972
|
-
type: Literal[
|
989
|
+
type: Literal["WaitTimeFromHeader"]
|
973
990
|
header: str = Field(
|
974
991
|
...,
|
975
|
-
description=
|
976
|
-
examples=[
|
977
|
-
title=
|
992
|
+
description="The name of the response header defining how long to wait before retrying.",
|
993
|
+
examples=["Retry-After"],
|
994
|
+
title="Response Header Name",
|
978
995
|
)
|
979
996
|
regex: Optional[str] = Field(
|
980
997
|
None,
|
981
|
-
description=
|
982
|
-
examples=[
|
983
|
-
title=
|
998
|
+
description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
|
999
|
+
examples=["([-+]?\\d+)"],
|
1000
|
+
title="Extraction Regex",
|
984
1001
|
)
|
985
1002
|
max_waiting_time_in_seconds: Optional[float] = Field(
|
986
1003
|
None,
|
987
|
-
description=
|
1004
|
+
description="Given the value extracted from the header is greater than this value, stop the stream.",
|
988
1005
|
examples=[3600],
|
989
|
-
title=
|
1006
|
+
title="Max Waiting Time in Seconds",
|
990
1007
|
)
|
991
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1008
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
992
1009
|
|
993
1010
|
|
994
1011
|
class WaitUntilTimeFromHeader(BaseModel):
|
995
|
-
type: Literal[
|
1012
|
+
type: Literal["WaitUntilTimeFromHeader"]
|
996
1013
|
header: str = Field(
|
997
1014
|
...,
|
998
|
-
description=
|
999
|
-
examples=[
|
1000
|
-
title=
|
1015
|
+
description="The name of the response header defining how long to wait before retrying.",
|
1016
|
+
examples=["wait_time"],
|
1017
|
+
title="Response Header",
|
1001
1018
|
)
|
1002
1019
|
min_wait: Optional[Union[float, str]] = Field(
|
1003
1020
|
None,
|
1004
|
-
description=
|
1005
|
-
examples=[10,
|
1006
|
-
title=
|
1021
|
+
description="Minimum time to wait before retrying.",
|
1022
|
+
examples=[10, "60"],
|
1023
|
+
title="Minimum Wait Time",
|
1007
1024
|
)
|
1008
1025
|
regex: Optional[str] = Field(
|
1009
1026
|
None,
|
1010
|
-
description=
|
1011
|
-
examples=[
|
1012
|
-
title=
|
1027
|
+
description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
|
1028
|
+
examples=["([-+]?\\d+)"],
|
1029
|
+
title="Extraction Regex",
|
1013
1030
|
)
|
1014
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1031
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1015
1032
|
|
1016
1033
|
|
1017
1034
|
class AddedFieldDefinition(BaseModel):
|
1018
|
-
type: Literal[
|
1035
|
+
type: Literal["AddedFieldDefinition"]
|
1019
1036
|
path: List[str] = Field(
|
1020
1037
|
...,
|
1021
|
-
description=
|
1022
|
-
examples=[[
|
1023
|
-
title=
|
1038
|
+
description="List of strings defining the path where to add the value on the record.",
|
1039
|
+
examples=[["segment_id"], ["metadata", "segment_id"]],
|
1040
|
+
title="Path",
|
1024
1041
|
)
|
1025
1042
|
value: str = Field(
|
1026
1043
|
...,
|
@@ -1030,167 +1047,167 @@ class AddedFieldDefinition(BaseModel):
|
|
1030
1047
|
"{{ record['MetaData']['LastUpdatedTime'] }}",
|
1031
1048
|
"{{ stream_partition['segment_id'] }}",
|
1032
1049
|
],
|
1033
|
-
title=
|
1050
|
+
title="Value",
|
1034
1051
|
)
|
1035
1052
|
value_type: Optional[ValueType] = Field(
|
1036
1053
|
None,
|
1037
|
-
description=
|
1038
|
-
title=
|
1054
|
+
description="Type of the value. If not specified, the type will be inferred from the value.",
|
1055
|
+
title="Value Type",
|
1039
1056
|
)
|
1040
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1057
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1041
1058
|
|
1042
1059
|
|
1043
1060
|
class AddFields(BaseModel):
|
1044
|
-
type: Literal[
|
1061
|
+
type: Literal["AddFields"]
|
1045
1062
|
fields: List[AddedFieldDefinition] = Field(
|
1046
1063
|
...,
|
1047
|
-
description=
|
1048
|
-
title=
|
1064
|
+
description="List of transformations (path and corresponding value) that will be added to the record.",
|
1065
|
+
title="Fields",
|
1049
1066
|
)
|
1050
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1067
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1051
1068
|
|
1052
1069
|
|
1053
1070
|
class ApiKeyAuthenticator(BaseModel):
|
1054
|
-
type: Literal[
|
1071
|
+
type: Literal["ApiKeyAuthenticator"]
|
1055
1072
|
api_token: Optional[str] = Field(
|
1056
1073
|
None,
|
1057
|
-
description=
|
1074
|
+
description="The API key to inject in the request. Fill it in the user inputs.",
|
1058
1075
|
examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"],
|
1059
|
-
title=
|
1076
|
+
title="API Key",
|
1060
1077
|
)
|
1061
1078
|
header: Optional[str] = Field(
|
1062
1079
|
None,
|
1063
|
-
description=
|
1064
|
-
examples=[
|
1065
|
-
title=
|
1080
|
+
description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.",
|
1081
|
+
examples=["Authorization", "Api-Token", "X-Auth-Token"],
|
1082
|
+
title="Header Name",
|
1066
1083
|
)
|
1067
1084
|
inject_into: Optional[RequestOption] = Field(
|
1068
1085
|
None,
|
1069
|
-
description=
|
1086
|
+
description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.",
|
1070
1087
|
examples=[
|
1071
|
-
{
|
1072
|
-
{
|
1088
|
+
{"inject_into": "header", "field_name": "Authorization"},
|
1089
|
+
{"inject_into": "request_parameter", "field_name": "authKey"},
|
1073
1090
|
],
|
1074
|
-
title=
|
1091
|
+
title="Inject API Key Into Outgoing HTTP Request",
|
1075
1092
|
)
|
1076
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1093
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1077
1094
|
|
1078
1095
|
|
1079
1096
|
class AuthFlow(BaseModel):
|
1080
1097
|
auth_flow_type: Optional[AuthFlowType] = Field(
|
1081
|
-
None, description=
|
1098
|
+
None, description="The type of auth to use", title="Auth flow type"
|
1082
1099
|
)
|
1083
1100
|
predicate_key: Optional[List[str]] = Field(
|
1084
1101
|
None,
|
1085
|
-
description=
|
1086
|
-
examples=[[
|
1087
|
-
title=
|
1102
|
+
description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.",
|
1103
|
+
examples=[["credentials", "auth_type"]],
|
1104
|
+
title="Predicate key",
|
1088
1105
|
)
|
1089
1106
|
predicate_value: Optional[str] = Field(
|
1090
1107
|
None,
|
1091
|
-
description=
|
1092
|
-
examples=[
|
1093
|
-
title=
|
1108
|
+
description="Value of the predicate_key fields for the advanced auth to be applicable.",
|
1109
|
+
examples=["Oauth"],
|
1110
|
+
title="Predicate value",
|
1094
1111
|
)
|
1095
1112
|
oauth_config_specification: Optional[OAuthConfigSpecification] = None
|
1096
1113
|
|
1097
1114
|
|
1098
1115
|
class DatetimeBasedCursor(BaseModel):
|
1099
|
-
type: Literal[
|
1116
|
+
type: Literal["DatetimeBasedCursor"]
|
1100
1117
|
cursor_field: str = Field(
|
1101
1118
|
...,
|
1102
|
-
description=
|
1103
|
-
examples=[
|
1104
|
-
title=
|
1119
|
+
description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.",
|
1120
|
+
examples=["created_at", "{{ config['record_cursor'] }}"],
|
1121
|
+
title="Cursor Field",
|
1105
1122
|
)
|
1106
1123
|
datetime_format: str = Field(
|
1107
1124
|
...,
|
1108
|
-
description=
|
1109
|
-
examples=[
|
1110
|
-
title=
|
1125
|
+
description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
|
1126
|
+
examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"],
|
1127
|
+
title="Outgoing Datetime Format",
|
1111
1128
|
)
|
1112
1129
|
start_datetime: Union[str, MinMaxDatetime] = Field(
|
1113
1130
|
...,
|
1114
|
-
description=
|
1115
|
-
examples=[
|
1116
|
-
title=
|
1131
|
+
description="The datetime that determines the earliest record that should be synced.",
|
1132
|
+
examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"],
|
1133
|
+
title="Start Datetime",
|
1117
1134
|
)
|
1118
1135
|
cursor_datetime_formats: Optional[List[str]] = Field(
|
1119
1136
|
None,
|
1120
|
-
description=
|
1121
|
-
title=
|
1137
|
+
description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.",
|
1138
|
+
title="Cursor Datetime Formats",
|
1122
1139
|
)
|
1123
1140
|
cursor_granularity: Optional[str] = Field(
|
1124
1141
|
None,
|
1125
|
-
description=
|
1126
|
-
examples=[
|
1127
|
-
title=
|
1142
|
+
description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.",
|
1143
|
+
examples=["PT1S"],
|
1144
|
+
title="Cursor Granularity",
|
1128
1145
|
)
|
1129
1146
|
end_datetime: Optional[Union[str, MinMaxDatetime]] = Field(
|
1130
1147
|
None,
|
1131
|
-
description=
|
1132
|
-
examples=[
|
1133
|
-
title=
|
1148
|
+
description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.",
|
1149
|
+
examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"],
|
1150
|
+
title="End Datetime",
|
1134
1151
|
)
|
1135
1152
|
end_time_option: Optional[RequestOption] = Field(
|
1136
1153
|
None,
|
1137
|
-
description=
|
1138
|
-
title=
|
1154
|
+
description="Optionally configures how the end datetime will be sent in requests to the source API.",
|
1155
|
+
title="Inject End Time Into Outgoing HTTP Request",
|
1139
1156
|
)
|
1140
1157
|
is_data_feed: Optional[bool] = Field(
|
1141
1158
|
None,
|
1142
|
-
description=
|
1143
|
-
title=
|
1159
|
+
description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.",
|
1160
|
+
title="Whether the target API is formatted as a data feed",
|
1144
1161
|
)
|
1145
1162
|
is_client_side_incremental: Optional[bool] = Field(
|
1146
1163
|
None,
|
1147
|
-
description=
|
1148
|
-
title=
|
1164
|
+
description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.",
|
1165
|
+
title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)",
|
1149
1166
|
)
|
1150
1167
|
is_compare_strictly: Optional[bool] = Field(
|
1151
1168
|
False,
|
1152
|
-
description=
|
1153
|
-
title=
|
1169
|
+
description="Set to True if the target API does not accept queries where the start time equal the end time.",
|
1170
|
+
title="Whether to skip requests if the start time equals the end time",
|
1154
1171
|
)
|
1155
1172
|
global_substream_cursor: Optional[bool] = Field(
|
1156
1173
|
False,
|
1157
|
-
description=
|
1158
|
-
title=
|
1174
|
+
description="This setting optimizes performance when the parent stream has thousands of partitions by storing the cursor as a single value rather than per partition. Notably, the substream state is updated only at the end of the sync, which helps prevent data loss in case of a sync failure. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/incremental-syncs).",
|
1175
|
+
title="Whether to store cursor as one value instead of per partition",
|
1159
1176
|
)
|
1160
1177
|
lookback_window: Optional[str] = Field(
|
1161
1178
|
None,
|
1162
|
-
description=
|
1163
|
-
examples=[
|
1164
|
-
title=
|
1179
|
+
description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.",
|
1180
|
+
examples=["P1D", "P{{ config['lookback_days'] }}D"],
|
1181
|
+
title="Lookback Window",
|
1165
1182
|
)
|
1166
1183
|
partition_field_end: Optional[str] = Field(
|
1167
1184
|
None,
|
1168
|
-
description=
|
1169
|
-
examples=[
|
1170
|
-
title=
|
1185
|
+
description="Name of the partition start time field.",
|
1186
|
+
examples=["ending_time"],
|
1187
|
+
title="Partition Field End",
|
1171
1188
|
)
|
1172
1189
|
partition_field_start: Optional[str] = Field(
|
1173
1190
|
None,
|
1174
|
-
description=
|
1175
|
-
examples=[
|
1176
|
-
title=
|
1191
|
+
description="Name of the partition end time field.",
|
1192
|
+
examples=["starting_time"],
|
1193
|
+
title="Partition Field Start",
|
1177
1194
|
)
|
1178
1195
|
start_time_option: Optional[RequestOption] = Field(
|
1179
1196
|
None,
|
1180
|
-
description=
|
1181
|
-
title=
|
1197
|
+
description="Optionally configures how the start datetime will be sent in requests to the source API.",
|
1198
|
+
title="Inject Start Time Into Outgoing HTTP Request",
|
1182
1199
|
)
|
1183
1200
|
step: Optional[str] = Field(
|
1184
1201
|
None,
|
1185
|
-
description=
|
1186
|
-
examples=[
|
1187
|
-
title=
|
1202
|
+
description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.",
|
1203
|
+
examples=["P1W", "{{ config['step_increment'] }}"],
|
1204
|
+
title="Step",
|
1188
1205
|
)
|
1189
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1206
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1190
1207
|
|
1191
1208
|
|
1192
1209
|
class DefaultErrorHandler(BaseModel):
|
1193
|
-
type: Literal[
|
1210
|
+
type: Literal["DefaultErrorHandler"]
|
1194
1211
|
backoff_strategies: Optional[
|
1195
1212
|
List[
|
1196
1213
|
Union[
|
@@ -1203,124 +1220,124 @@ class DefaultErrorHandler(BaseModel):
|
|
1203
1220
|
]
|
1204
1221
|
] = Field(
|
1205
1222
|
None,
|
1206
|
-
description=
|
1207
|
-
title=
|
1223
|
+
description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.",
|
1224
|
+
title="Backoff Strategies",
|
1208
1225
|
)
|
1209
1226
|
max_retries: Optional[int] = Field(
|
1210
1227
|
5,
|
1211
|
-
description=
|
1228
|
+
description="The maximum number of time to retry a retryable request before giving up and failing.",
|
1212
1229
|
examples=[5, 0, 10],
|
1213
|
-
title=
|
1230
|
+
title="Max Retry Count",
|
1214
1231
|
)
|
1215
1232
|
response_filters: Optional[List[HttpResponseFilter]] = Field(
|
1216
1233
|
None,
|
1217
1234
|
description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.",
|
1218
|
-
title=
|
1235
|
+
title="Response Filters",
|
1219
1236
|
)
|
1220
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1237
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1221
1238
|
|
1222
1239
|
|
1223
1240
|
class DefaultPaginator(BaseModel):
|
1224
|
-
type: Literal[
|
1241
|
+
type: Literal["DefaultPaginator"]
|
1225
1242
|
pagination_strategy: Union[
|
1226
1243
|
CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement
|
1227
1244
|
] = Field(
|
1228
1245
|
...,
|
1229
|
-
description=
|
1230
|
-
title=
|
1246
|
+
description="Strategy defining how records are paginated.",
|
1247
|
+
title="Pagination Strategy",
|
1231
1248
|
)
|
1232
1249
|
page_size_option: Optional[RequestOption] = None
|
1233
1250
|
page_token_option: Optional[Union[RequestOption, RequestPath]] = None
|
1234
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1251
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1235
1252
|
|
1236
1253
|
|
1237
1254
|
class SessionTokenRequestApiKeyAuthenticator(BaseModel):
|
1238
|
-
type: Literal[
|
1255
|
+
type: Literal["ApiKey"]
|
1239
1256
|
inject_into: RequestOption = Field(
|
1240
1257
|
...,
|
1241
|
-
description=
|
1258
|
+
description="Configure how the API Key will be sent in requests to the source API.",
|
1242
1259
|
examples=[
|
1243
|
-
{
|
1244
|
-
{
|
1260
|
+
{"inject_into": "header", "field_name": "Authorization"},
|
1261
|
+
{"inject_into": "request_parameter", "field_name": "authKey"},
|
1245
1262
|
],
|
1246
|
-
title=
|
1263
|
+
title="Inject API Key Into Outgoing HTTP Request",
|
1247
1264
|
)
|
1248
1265
|
|
1249
1266
|
|
1250
1267
|
class ListPartitionRouter(BaseModel):
|
1251
|
-
type: Literal[
|
1268
|
+
type: Literal["ListPartitionRouter"]
|
1252
1269
|
cursor_field: str = Field(
|
1253
1270
|
...,
|
1254
1271
|
description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.',
|
1255
|
-
examples=[
|
1256
|
-
title=
|
1272
|
+
examples=["section", "{{ config['section_key'] }}"],
|
1273
|
+
title="Current Partition Value Identifier",
|
1257
1274
|
)
|
1258
1275
|
values: Union[str, List[str]] = Field(
|
1259
1276
|
...,
|
1260
|
-
description=
|
1261
|
-
examples=[[
|
1262
|
-
title=
|
1277
|
+
description="The list of attributes being iterated over and used as input for the requests made to the source API.",
|
1278
|
+
examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"],
|
1279
|
+
title="Partition Values",
|
1263
1280
|
)
|
1264
1281
|
request_option: Optional[RequestOption] = Field(
|
1265
1282
|
None,
|
1266
|
-
description=
|
1267
|
-
title=
|
1283
|
+
description="A request option describing where the list value should be injected into and under what field name if applicable.",
|
1284
|
+
title="Inject Partition Value Into Outgoing HTTP Request",
|
1268
1285
|
)
|
1269
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1286
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1270
1287
|
|
1271
1288
|
|
1272
1289
|
class RecordSelector(BaseModel):
|
1273
|
-
type: Literal[
|
1290
|
+
type: Literal["RecordSelector"]
|
1274
1291
|
extractor: Union[CustomRecordExtractor, DpathExtractor]
|
1275
1292
|
record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field(
|
1276
1293
|
None,
|
1277
|
-
description=
|
1278
|
-
title=
|
1294
|
+
description="Responsible for filtering records to be emitted by the Source.",
|
1295
|
+
title="Record Filter",
|
1279
1296
|
)
|
1280
1297
|
schema_normalization: Optional[SchemaNormalization] = SchemaNormalization.None_
|
1281
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1298
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1282
1299
|
|
1283
1300
|
|
1284
1301
|
class Spec(BaseModel):
|
1285
|
-
type: Literal[
|
1302
|
+
type: Literal["Spec"]
|
1286
1303
|
connection_specification: Dict[str, Any] = Field(
|
1287
1304
|
...,
|
1288
|
-
description=
|
1289
|
-
title=
|
1305
|
+
description="A connection specification describing how a the connector can be configured.",
|
1306
|
+
title="Connection Specification",
|
1290
1307
|
)
|
1291
1308
|
documentation_url: Optional[str] = Field(
|
1292
1309
|
None,
|
1293
1310
|
description="URL of the connector's documentation page.",
|
1294
|
-
examples=[
|
1295
|
-
title=
|
1311
|
+
examples=["https://docs.airbyte.com/integrations/sources/dremio"],
|
1312
|
+
title="Documentation URL",
|
1296
1313
|
)
|
1297
1314
|
advanced_auth: Optional[AuthFlow] = Field(
|
1298
1315
|
None,
|
1299
|
-
description=
|
1300
|
-
title=
|
1316
|
+
description="Advanced specification for configuring the authentication flow.",
|
1317
|
+
title="Advanced Auth",
|
1301
1318
|
)
|
1302
1319
|
|
1303
1320
|
|
1304
1321
|
class CompositeErrorHandler(BaseModel):
|
1305
|
-
type: Literal[
|
1322
|
+
type: Literal["CompositeErrorHandler"]
|
1306
1323
|
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field(
|
1307
1324
|
...,
|
1308
|
-
description=
|
1309
|
-
title=
|
1325
|
+
description="List of error handlers to iterate on to determine how to handle a failed response.",
|
1326
|
+
title="Error Handlers",
|
1310
1327
|
)
|
1311
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1328
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1312
1329
|
|
1313
1330
|
|
1314
1331
|
class DeclarativeSource(BaseModel):
|
1315
1332
|
class Config:
|
1316
1333
|
extra = Extra.forbid
|
1317
1334
|
|
1318
|
-
type: Literal[
|
1335
|
+
type: Literal["DeclarativeSource"]
|
1319
1336
|
check: CheckStream
|
1320
1337
|
streams: List[DeclarativeStream]
|
1321
1338
|
version: str = Field(
|
1322
1339
|
...,
|
1323
|
-
description=
|
1340
|
+
description="The version of the Airbyte CDK used to build and test the source.",
|
1324
1341
|
)
|
1325
1342
|
schemas: Optional[Schemas] = None
|
1326
1343
|
definitions: Optional[Dict[str, Any]] = None
|
@@ -1328,11 +1345,11 @@ class DeclarativeSource(BaseModel):
|
|
1328
1345
|
concurrency_level: Optional[ConcurrencyLevel] = None
|
1329
1346
|
metadata: Optional[Dict[str, Any]] = Field(
|
1330
1347
|
None,
|
1331
|
-
description=
|
1348
|
+
description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
|
1332
1349
|
)
|
1333
1350
|
description: Optional[str] = Field(
|
1334
1351
|
None,
|
1335
|
-
description=
|
1352
|
+
description="A description of the connector. It will be presented on the Source documentation page.",
|
1336
1353
|
)
|
1337
1354
|
|
1338
1355
|
|
@@ -1340,12 +1357,12 @@ class SelectiveAuthenticator(BaseModel):
|
|
1340
1357
|
class Config:
|
1341
1358
|
extra = Extra.allow
|
1342
1359
|
|
1343
|
-
type: Literal[
|
1360
|
+
type: Literal["SelectiveAuthenticator"]
|
1344
1361
|
authenticator_selection_path: List[str] = Field(
|
1345
1362
|
...,
|
1346
|
-
description=
|
1347
|
-
examples=[[
|
1348
|
-
title=
|
1363
|
+
description="Path of the field in config with selected authenticator name",
|
1364
|
+
examples=[["auth"], ["auth", "type"]],
|
1365
|
+
title="Authenticator Selection Path",
|
1349
1366
|
)
|
1350
1367
|
authenticators: Dict[
|
1351
1368
|
str,
|
@@ -1362,132 +1379,128 @@ class SelectiveAuthenticator(BaseModel):
|
|
1362
1379
|
],
|
1363
1380
|
] = Field(
|
1364
1381
|
...,
|
1365
|
-
description=
|
1382
|
+
description="Authenticators to select from.",
|
1366
1383
|
examples=[
|
1367
1384
|
{
|
1368
|
-
|
1369
|
-
|
1370
|
-
|
1371
|
-
|
1385
|
+
"authenticators": {
|
1386
|
+
"token": "#/definitions/ApiKeyAuthenticator",
|
1387
|
+
"oauth": "#/definitions/OAuthAuthenticator",
|
1388
|
+
"jwt": "#/definitions/JwtAuthenticator",
|
1372
1389
|
}
|
1373
1390
|
}
|
1374
1391
|
],
|
1375
|
-
title=
|
1392
|
+
title="Authenticators",
|
1376
1393
|
)
|
1377
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1394
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1378
1395
|
|
1379
1396
|
|
1380
1397
|
class DeclarativeStream(BaseModel):
|
1381
1398
|
class Config:
|
1382
1399
|
extra = Extra.allow
|
1383
1400
|
|
1384
|
-
type: Literal[
|
1401
|
+
type: Literal["DeclarativeStream"]
|
1385
1402
|
retriever: Union[AsyncRetriever, CustomRetriever, SimpleRetriever] = Field(
|
1386
1403
|
...,
|
1387
|
-
description=
|
1388
|
-
title=
|
1404
|
+
description="Component used to coordinate how records are extracted across stream slices and request pages.",
|
1405
|
+
title="Retriever",
|
1389
1406
|
)
|
1390
|
-
incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = (
|
1391
|
-
|
1392
|
-
|
1393
|
-
|
1394
|
-
title='Incremental Sync',
|
1395
|
-
)
|
1396
|
-
)
|
1397
|
-
name: Optional[str] = Field(
|
1398
|
-
'', description='The stream name.', example=['Users'], title='Name'
|
1407
|
+
incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field(
|
1408
|
+
None,
|
1409
|
+
description="Component used to fetch data incrementally based on a time field in the data.",
|
1410
|
+
title="Incremental Sync",
|
1399
1411
|
)
|
1412
|
+
name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name")
|
1400
1413
|
primary_key: Optional[PrimaryKey] = Field(
|
1401
|
-
|
1414
|
+
"", description="The primary key of the stream.", title="Primary Key"
|
1402
1415
|
)
|
1403
|
-
schema_loader: Optional[
|
1404
|
-
|
1405
|
-
|
1406
|
-
|
1407
|
-
|
1408
|
-
|
1416
|
+
schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]] = (
|
1417
|
+
Field(
|
1418
|
+
None,
|
1419
|
+
description="Component used to retrieve the schema for the current stream.",
|
1420
|
+
title="Schema Loader",
|
1421
|
+
)
|
1409
1422
|
)
|
1410
1423
|
transformations: Optional[
|
1411
1424
|
List[Union[AddFields, CustomTransformation, RemoveFields, KeysToLower]]
|
1412
1425
|
] = Field(
|
1413
1426
|
None,
|
1414
|
-
description=
|
1415
|
-
title=
|
1427
|
+
description="A list of transformations to be applied to each output record.",
|
1428
|
+
title="Transformations",
|
1416
1429
|
)
|
1417
1430
|
state_migrations: Optional[
|
1418
1431
|
List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]
|
1419
1432
|
] = Field(
|
1420
1433
|
[],
|
1421
|
-
description=
|
1422
|
-
title=
|
1434
|
+
description="Array of state migrations to be applied on the input state",
|
1435
|
+
title="State Migrations",
|
1423
1436
|
)
|
1424
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1437
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1425
1438
|
|
1426
1439
|
|
1427
1440
|
class SessionTokenAuthenticator(BaseModel):
|
1428
|
-
type: Literal[
|
1441
|
+
type: Literal["SessionTokenAuthenticator"]
|
1429
1442
|
login_requester: HttpRequester = Field(
|
1430
1443
|
...,
|
1431
|
-
description=
|
1444
|
+
description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.",
|
1432
1445
|
examples=[
|
1433
1446
|
{
|
1434
|
-
|
1435
|
-
|
1436
|
-
|
1437
|
-
|
1438
|
-
|
1439
|
-
|
1440
|
-
|
1447
|
+
"type": "HttpRequester",
|
1448
|
+
"url_base": "https://my_api.com",
|
1449
|
+
"path": "/login",
|
1450
|
+
"authenticator": {
|
1451
|
+
"type": "BasicHttpAuthenticator",
|
1452
|
+
"username": "{{ config.username }}",
|
1453
|
+
"password": "{{ config.password }}",
|
1441
1454
|
},
|
1442
1455
|
}
|
1443
1456
|
],
|
1444
|
-
title=
|
1457
|
+
title="Login Requester",
|
1445
1458
|
)
|
1446
1459
|
session_token_path: List[str] = Field(
|
1447
1460
|
...,
|
1448
|
-
description=
|
1449
|
-
examples=[[
|
1450
|
-
title=
|
1461
|
+
description="The path in the response body returned from the login requester to the session token.",
|
1462
|
+
examples=[["access_token"], ["result", "token"]],
|
1463
|
+
title="Session Token Path",
|
1451
1464
|
)
|
1452
1465
|
expiration_duration: Optional[str] = Field(
|
1453
1466
|
None,
|
1454
|
-
description=
|
1455
|
-
examples=[
|
1456
|
-
title=
|
1467
|
+
description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.",
|
1468
|
+
examples=["PT1H", "P1D"],
|
1469
|
+
title="Expiration Duration",
|
1457
1470
|
)
|
1458
1471
|
request_authentication: Union[
|
1459
1472
|
SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator
|
1460
1473
|
] = Field(
|
1461
1474
|
...,
|
1462
|
-
description=
|
1463
|
-
title=
|
1475
|
+
description="Authentication method to use for requests sent to the API, specifying how to inject the session token.",
|
1476
|
+
title="Data Request Authentication",
|
1464
1477
|
)
|
1465
1478
|
decoder: Optional[Union[JsonDecoder, XmlDecoder]] = Field(
|
1466
|
-
None, description=
|
1479
|
+
None, description="Component used to decode the response.", title="Decoder"
|
1467
1480
|
)
|
1468
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1481
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1469
1482
|
|
1470
1483
|
|
1471
1484
|
class HttpRequester(BaseModel):
|
1472
|
-
type: Literal[
|
1485
|
+
type: Literal["HttpRequester"]
|
1473
1486
|
url_base: str = Field(
|
1474
1487
|
...,
|
1475
|
-
description=
|
1488
|
+
description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
|
1476
1489
|
examples=[
|
1477
|
-
|
1490
|
+
"https://connect.squareup.com/v2",
|
1478
1491
|
"{{ config['base_url'] or 'https://app.posthog.com'}}/api/",
|
1479
1492
|
],
|
1480
|
-
title=
|
1493
|
+
title="API Base URL",
|
1481
1494
|
)
|
1482
1495
|
path: str = Field(
|
1483
1496
|
...,
|
1484
|
-
description=
|
1497
|
+
description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
|
1485
1498
|
examples=[
|
1486
|
-
|
1499
|
+
"/products",
|
1487
1500
|
"/quotes/{{ stream_partition['id'] }}/quote_line_groups",
|
1488
1501
|
"/trades/{{ config['symbol_id'] }}/history",
|
1489
1502
|
],
|
1490
|
-
title=
|
1503
|
+
title="URL Path",
|
1491
1504
|
)
|
1492
1505
|
authenticator: Optional[
|
1493
1506
|
Union[
|
@@ -1504,111 +1517,111 @@ class HttpRequester(BaseModel):
|
|
1504
1517
|
]
|
1505
1518
|
] = Field(
|
1506
1519
|
None,
|
1507
|
-
description=
|
1508
|
-
title=
|
1520
|
+
description="Authentication method to use for requests sent to the API.",
|
1521
|
+
title="Authenticator",
|
1509
1522
|
)
|
1510
1523
|
error_handler: Optional[
|
1511
1524
|
Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]
|
1512
1525
|
] = Field(
|
1513
1526
|
None,
|
1514
|
-
description=
|
1515
|
-
title=
|
1527
|
+
description="Error handler component that defines how to handle errors.",
|
1528
|
+
title="Error Handler",
|
1516
1529
|
)
|
1517
1530
|
http_method: Optional[HttpMethod] = Field(
|
1518
1531
|
HttpMethod.GET,
|
1519
|
-
description=
|
1520
|
-
examples=[
|
1521
|
-
title=
|
1532
|
+
description="The HTTP method used to fetch data from the source (can be GET or POST).",
|
1533
|
+
examples=["GET", "POST"],
|
1534
|
+
title="HTTP Method",
|
1522
1535
|
)
|
1523
1536
|
request_body_data: Optional[Union[str, Dict[str, str]]] = Field(
|
1524
1537
|
None,
|
1525
|
-
description=
|
1538
|
+
description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.",
|
1526
1539
|
examples=[
|
1527
1540
|
'[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n'
|
1528
1541
|
],
|
1529
|
-
title=
|
1542
|
+
title="Request Body Payload (Non-JSON)",
|
1530
1543
|
)
|
1531
1544
|
request_body_json: Optional[Union[str, Dict[str, Any]]] = Field(
|
1532
1545
|
None,
|
1533
|
-
description=
|
1546
|
+
description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.",
|
1534
1547
|
examples=[
|
1535
|
-
{
|
1536
|
-
{
|
1537
|
-
{
|
1548
|
+
{"sort_order": "ASC", "sort_field": "CREATED_AT"},
|
1549
|
+
{"key": "{{ config['value'] }}"},
|
1550
|
+
{"sort": {"field": "updated_at", "order": "ascending"}},
|
1538
1551
|
],
|
1539
|
-
title=
|
1552
|
+
title="Request Body JSON Payload",
|
1540
1553
|
)
|
1541
1554
|
request_headers: Optional[Union[str, Dict[str, str]]] = Field(
|
1542
1555
|
None,
|
1543
|
-
description=
|
1544
|
-
examples=[{
|
1545
|
-
title=
|
1556
|
+
description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.",
|
1557
|
+
examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
|
1558
|
+
title="Request Headers",
|
1546
1559
|
)
|
1547
1560
|
request_parameters: Optional[Union[str, Dict[str, str]]] = Field(
|
1548
1561
|
None,
|
1549
|
-
description=
|
1562
|
+
description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
|
1550
1563
|
examples=[
|
1551
|
-
{
|
1564
|
+
{"unit": "day"},
|
1552
1565
|
{
|
1553
|
-
|
1566
|
+
"query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
|
1554
1567
|
},
|
1555
|
-
{
|
1556
|
-
{
|
1568
|
+
{"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
|
1569
|
+
{"sort_by[asc]": "updated_at"},
|
1557
1570
|
],
|
1558
|
-
title=
|
1571
|
+
title="Query Parameters",
|
1559
1572
|
)
|
1560
1573
|
use_cache: Optional[bool] = Field(
|
1561
1574
|
False,
|
1562
|
-
description=
|
1563
|
-
title=
|
1575
|
+
description="Enables stream requests caching. This field is automatically set by the CDK.",
|
1576
|
+
title="Use Cache",
|
1564
1577
|
)
|
1565
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1578
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1566
1579
|
|
1567
1580
|
|
1568
1581
|
class ParentStreamConfig(BaseModel):
|
1569
|
-
type: Literal[
|
1582
|
+
type: Literal["ParentStreamConfig"]
|
1570
1583
|
parent_key: str = Field(
|
1571
1584
|
...,
|
1572
|
-
description=
|
1573
|
-
examples=[
|
1574
|
-
title=
|
1585
|
+
description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.",
|
1586
|
+
examples=["id", "{{ config['parent_record_id'] }}"],
|
1587
|
+
title="Parent Key",
|
1575
1588
|
)
|
1576
1589
|
stream: DeclarativeStream = Field(
|
1577
|
-
..., description=
|
1590
|
+
..., description="Reference to the parent stream.", title="Parent Stream"
|
1578
1591
|
)
|
1579
1592
|
partition_field: str = Field(
|
1580
1593
|
...,
|
1581
|
-
description=
|
1582
|
-
examples=[
|
1583
|
-
title=
|
1594
|
+
description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.",
|
1595
|
+
examples=["parent_id", "{{ config['parent_partition_field'] }}"],
|
1596
|
+
title="Current Parent Key Value Identifier",
|
1584
1597
|
)
|
1585
1598
|
request_option: Optional[RequestOption] = Field(
|
1586
1599
|
None,
|
1587
|
-
description=
|
1588
|
-
title=
|
1600
|
+
description="A request option describing where the parent key value should be injected into and under what field name if applicable.",
|
1601
|
+
title="Request Option",
|
1589
1602
|
)
|
1590
1603
|
incremental_dependency: Optional[bool] = Field(
|
1591
1604
|
False,
|
1592
|
-
description=
|
1593
|
-
title=
|
1605
|
+
description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.",
|
1606
|
+
title="Incremental Dependency",
|
1594
1607
|
)
|
1595
1608
|
extra_fields: Optional[List[List[str]]] = Field(
|
1596
1609
|
None,
|
1597
|
-
description=
|
1598
|
-
title=
|
1610
|
+
description="Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.",
|
1611
|
+
title="Extra Fields",
|
1599
1612
|
)
|
1600
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1613
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1601
1614
|
|
1602
1615
|
|
1603
1616
|
class SimpleRetriever(BaseModel):
|
1604
|
-
type: Literal[
|
1617
|
+
type: Literal["SimpleRetriever"]
|
1605
1618
|
record_selector: RecordSelector = Field(
|
1606
1619
|
...,
|
1607
|
-
description=
|
1620
|
+
description="Component that describes how to extract records from a HTTP response.",
|
1608
1621
|
)
|
1609
1622
|
requester: Union[CustomRequester, HttpRequester] = Field(
|
1610
1623
|
...,
|
1611
|
-
description=
|
1624
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API.",
|
1612
1625
|
)
|
1613
1626
|
paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
|
1614
1627
|
None,
|
@@ -1616,61 +1629,64 @@ class SimpleRetriever(BaseModel):
|
|
1616
1629
|
)
|
1617
1630
|
ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field(
|
1618
1631
|
False,
|
1619
|
-
description=
|
1632
|
+
description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.",
|
1620
1633
|
)
|
1621
1634
|
partition_router: Optional[
|
1622
1635
|
Union[
|
1623
1636
|
CustomPartitionRouter,
|
1624
1637
|
ListPartitionRouter,
|
1625
1638
|
SubstreamPartitionRouter,
|
1626
|
-
List[
|
1627
|
-
Union[
|
1628
|
-
CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
|
1629
|
-
]
|
1630
|
-
],
|
1639
|
+
List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
|
1631
1640
|
]
|
1632
1641
|
] = Field(
|
1633
1642
|
[],
|
1634
|
-
description=
|
1635
|
-
title=
|
1643
|
+
description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
|
1644
|
+
title="Partition Router",
|
1636
1645
|
)
|
1637
|
-
decoder: Optional[
|
1638
|
-
|
1639
|
-
|
1640
|
-
|
1641
|
-
|
1642
|
-
|
1646
|
+
decoder: Optional[
|
1647
|
+
Union[
|
1648
|
+
CustomDecoder,
|
1649
|
+
JsonDecoder,
|
1650
|
+
JsonlDecoder,
|
1651
|
+
IterableDecoder,
|
1652
|
+
XmlDecoder,
|
1653
|
+
GzipJsonDecoder,
|
1654
|
+
]
|
1655
|
+
] = Field(
|
1656
|
+
None,
|
1657
|
+
description="Component decoding the response so records can be extracted.",
|
1658
|
+
title="Decoder",
|
1643
1659
|
)
|
1644
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1660
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1645
1661
|
|
1646
1662
|
|
1647
1663
|
class AsyncRetriever(BaseModel):
|
1648
|
-
type: Literal[
|
1664
|
+
type: Literal["AsyncRetriever"]
|
1649
1665
|
record_selector: RecordSelector = Field(
|
1650
1666
|
...,
|
1651
|
-
description=
|
1667
|
+
description="Component that describes how to extract records from a HTTP response.",
|
1652
1668
|
)
|
1653
1669
|
status_mapping: AsyncJobStatusMap = Field(
|
1654
|
-
..., description=
|
1670
|
+
..., description="Async Job Status to Airbyte CDK Async Job Status mapping."
|
1655
1671
|
)
|
1656
1672
|
status_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
|
1657
|
-
..., description=
|
1673
|
+
..., description="Responsible for fetching the actual status of the async job."
|
1658
1674
|
)
|
1659
1675
|
urls_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
|
1660
1676
|
...,
|
1661
|
-
description=
|
1677
|
+
description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.",
|
1662
1678
|
)
|
1663
1679
|
creation_requester: Union[CustomRequester, HttpRequester] = Field(
|
1664
1680
|
...,
|
1665
|
-
description=
|
1681
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.",
|
1666
1682
|
)
|
1667
1683
|
polling_requester: Union[CustomRequester, HttpRequester] = Field(
|
1668
1684
|
...,
|
1669
|
-
description=
|
1685
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.",
|
1670
1686
|
)
|
1671
1687
|
download_requester: Union[CustomRequester, HttpRequester] = Field(
|
1672
1688
|
...,
|
1673
|
-
description=
|
1689
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to download the data provided by the completed async job.",
|
1674
1690
|
)
|
1675
1691
|
download_paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
|
1676
1692
|
None,
|
@@ -1682,42 +1698,45 @@ class AsyncRetriever(BaseModel):
|
|
1682
1698
|
)
|
1683
1699
|
delete_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
|
1684
1700
|
None,
|
1685
|
-
description=
|
1701
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API to delete a job once the records are extracted.",
|
1686
1702
|
)
|
1687
1703
|
partition_router: Optional[
|
1688
1704
|
Union[
|
1689
1705
|
CustomPartitionRouter,
|
1690
1706
|
ListPartitionRouter,
|
1691
1707
|
SubstreamPartitionRouter,
|
1692
|
-
List[
|
1693
|
-
Union[
|
1694
|
-
CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
|
1695
|
-
]
|
1696
|
-
],
|
1708
|
+
List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
|
1697
1709
|
]
|
1698
1710
|
] = Field(
|
1699
1711
|
[],
|
1700
|
-
description=
|
1701
|
-
title=
|
1712
|
+
description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
|
1713
|
+
title="Partition Router",
|
1702
1714
|
)
|
1703
|
-
decoder: Optional[
|
1704
|
-
|
1705
|
-
|
1706
|
-
|
1707
|
-
|
1708
|
-
|
1715
|
+
decoder: Optional[
|
1716
|
+
Union[
|
1717
|
+
CustomDecoder,
|
1718
|
+
JsonDecoder,
|
1719
|
+
JsonlDecoder,
|
1720
|
+
IterableDecoder,
|
1721
|
+
XmlDecoder,
|
1722
|
+
GzipJsonDecoder,
|
1723
|
+
]
|
1724
|
+
] = Field(
|
1725
|
+
None,
|
1726
|
+
description="Component decoding the response so records can be extracted.",
|
1727
|
+
title="Decoder",
|
1709
1728
|
)
|
1710
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1729
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1711
1730
|
|
1712
1731
|
|
1713
1732
|
class SubstreamPartitionRouter(BaseModel):
|
1714
|
-
type: Literal[
|
1733
|
+
type: Literal["SubstreamPartitionRouter"]
|
1715
1734
|
parent_stream_configs: List[ParentStreamConfig] = Field(
|
1716
1735
|
...,
|
1717
|
-
description=
|
1718
|
-
title=
|
1736
|
+
description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.",
|
1737
|
+
title="Parent Stream Configs",
|
1719
1738
|
)
|
1720
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1739
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1721
1740
|
|
1722
1741
|
|
1723
1742
|
CompositeErrorHandler.update_forward_refs()
|