airbyte-cdk 0.67.0__py3-none-any.whl → 0.67.2__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- airbyte_cdk/sources/abstract_source.py +30 -69
- airbyte_cdk/sources/connector_state_manager.py +12 -26
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +552 -524
- airbyte_cdk/sources/file_based/config/csv_format.py +2 -0
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +32 -14
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +3 -19
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -3
- airbyte_cdk/sources/streams/__init__.py +2 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +3 -19
- airbyte_cdk/sources/streams/concurrent/cursor.py +1 -3
- airbyte_cdk/sources/streams/core.py +36 -34
- {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/METADATA +3 -2
- {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/RECORD +31 -31
- unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +2 -1
- unit_tests/sources/file_based/config/test_csv_format.py +6 -1
- unit_tests/sources/file_based/file_types/test_parquet_parser.py +51 -6
- unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +139 -199
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +91 -133
- unit_tests/sources/file_based/stream/concurrent/test_adapters.py +2 -13
- unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +2 -2
- unit_tests/sources/file_based/test_scenarios.py +2 -2
- unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py +9 -9
- unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py +5 -5
- unit_tests/sources/streams/concurrent/test_adapters.py +2 -13
- unit_tests/sources/streams/test_stream_read.py +221 -11
- unit_tests/sources/test_abstract_source.py +142 -130
- unit_tests/sources/test_connector_state_manager.py +3 -124
- unit_tests/sources/test_source.py +18 -14
- {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/WHEEL +0 -0
- {airbyte_cdk-0.67.0.dist-info → airbyte_cdk-0.67.2.dist-info}/top_level.txt +0 -0
@@ -11,145 +11,145 @@ from typing_extensions import Literal
|
|
11
11
|
|
12
12
|
|
13
13
|
class AuthFlowType(Enum):
|
14
|
-
oauth2_0 =
|
15
|
-
oauth1_0 =
|
14
|
+
oauth2_0 = 'oauth2.0'
|
15
|
+
oauth1_0 = 'oauth1.0'
|
16
16
|
|
17
17
|
|
18
18
|
class BasicHttpAuthenticator(BaseModel):
|
19
|
-
type: Literal[
|
19
|
+
type: Literal['BasicHttpAuthenticator']
|
20
20
|
username: str = Field(
|
21
21
|
...,
|
22
|
-
description=
|
22
|
+
description='The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.',
|
23
23
|
examples=["{{ config['username'] }}", "{{ config['api_key'] }}"],
|
24
|
-
title=
|
24
|
+
title='Username',
|
25
25
|
)
|
26
26
|
password: Optional[str] = Field(
|
27
|
-
|
28
|
-
description=
|
29
|
-
examples=["{{ config['password'] }}",
|
30
|
-
title=
|
27
|
+
'',
|
28
|
+
description='The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.',
|
29
|
+
examples=["{{ config['password'] }}", ''],
|
30
|
+
title='Password',
|
31
31
|
)
|
32
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
32
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
33
33
|
|
34
34
|
|
35
35
|
class BearerAuthenticator(BaseModel):
|
36
|
-
type: Literal[
|
36
|
+
type: Literal['BearerAuthenticator']
|
37
37
|
api_token: str = Field(
|
38
38
|
...,
|
39
|
-
description=
|
39
|
+
description='Token to inject as request header for authenticating with the API.',
|
40
40
|
examples=["{{ config['api_key'] }}", "{{ config['token'] }}"],
|
41
|
-
title=
|
41
|
+
title='Bearer Token',
|
42
42
|
)
|
43
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
43
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
44
44
|
|
45
45
|
|
46
46
|
class CheckStream(BaseModel):
|
47
|
-
type: Literal[
|
47
|
+
type: Literal['CheckStream']
|
48
48
|
stream_names: List[str] = Field(
|
49
49
|
...,
|
50
|
-
description=
|
51
|
-
examples=[[
|
52
|
-
title=
|
50
|
+
description='Names of the streams to try reading from when running a check operation.',
|
51
|
+
examples=[['users'], ['users', 'contacts']],
|
52
|
+
title='Stream Names',
|
53
53
|
)
|
54
54
|
|
55
55
|
|
56
56
|
class ConstantBackoffStrategy(BaseModel):
|
57
|
-
type: Literal[
|
57
|
+
type: Literal['ConstantBackoffStrategy']
|
58
58
|
backoff_time_in_seconds: Union[float, str] = Field(
|
59
59
|
...,
|
60
|
-
description=
|
60
|
+
description='Backoff time in seconds.',
|
61
61
|
examples=[30, 30.5, "{{ config['backoff_time'] }}"],
|
62
|
-
title=
|
62
|
+
title='Backoff Time',
|
63
63
|
)
|
64
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
64
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
65
65
|
|
66
66
|
|
67
67
|
class CustomAuthenticator(BaseModel):
|
68
68
|
class Config:
|
69
69
|
extra = Extra.allow
|
70
70
|
|
71
|
-
type: Literal[
|
71
|
+
type: Literal['CustomAuthenticator']
|
72
72
|
class_name: str = Field(
|
73
73
|
...,
|
74
|
-
description=
|
75
|
-
examples=[
|
76
|
-
title=
|
74
|
+
description='Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.',
|
75
|
+
examples=['source_railz.components.ShortLivedTokenAuthenticator'],
|
76
|
+
title='Class Name',
|
77
77
|
)
|
78
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
78
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
79
79
|
|
80
80
|
|
81
81
|
class CustomBackoffStrategy(BaseModel):
|
82
82
|
class Config:
|
83
83
|
extra = Extra.allow
|
84
84
|
|
85
|
-
type: Literal[
|
85
|
+
type: Literal['CustomBackoffStrategy']
|
86
86
|
class_name: str = Field(
|
87
87
|
...,
|
88
|
-
description=
|
89
|
-
examples=[
|
90
|
-
title=
|
88
|
+
description='Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.',
|
89
|
+
examples=['source_railz.components.MyCustomBackoffStrategy'],
|
90
|
+
title='Class Name',
|
91
91
|
)
|
92
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
92
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
93
93
|
|
94
94
|
|
95
95
|
class CustomErrorHandler(BaseModel):
|
96
96
|
class Config:
|
97
97
|
extra = Extra.allow
|
98
98
|
|
99
|
-
type: Literal[
|
99
|
+
type: Literal['CustomErrorHandler']
|
100
100
|
class_name: str = Field(
|
101
101
|
...,
|
102
|
-
description=
|
103
|
-
examples=[
|
104
|
-
title=
|
102
|
+
description='Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.',
|
103
|
+
examples=['source_railz.components.MyCustomErrorHandler'],
|
104
|
+
title='Class Name',
|
105
105
|
)
|
106
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
106
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
107
107
|
|
108
108
|
|
109
109
|
class CustomIncrementalSync(BaseModel):
|
110
110
|
class Config:
|
111
111
|
extra = Extra.allow
|
112
112
|
|
113
|
-
type: Literal[
|
113
|
+
type: Literal['CustomIncrementalSync']
|
114
114
|
class_name: str = Field(
|
115
115
|
...,
|
116
|
-
description=
|
117
|
-
examples=[
|
118
|
-
title=
|
116
|
+
description='Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.',
|
117
|
+
examples=['source_railz.components.MyCustomIncrementalSync'],
|
118
|
+
title='Class Name',
|
119
119
|
)
|
120
120
|
cursor_field: str = Field(
|
121
121
|
...,
|
122
|
-
description=
|
122
|
+
description='The location of the value on a record that will be used as a bookmark during sync.',
|
123
123
|
)
|
124
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
124
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
125
125
|
|
126
126
|
|
127
127
|
class CustomPaginationStrategy(BaseModel):
|
128
128
|
class Config:
|
129
129
|
extra = Extra.allow
|
130
130
|
|
131
|
-
type: Literal[
|
131
|
+
type: Literal['CustomPaginationStrategy']
|
132
132
|
class_name: str = Field(
|
133
133
|
...,
|
134
|
-
description=
|
135
|
-
examples=[
|
136
|
-
title=
|
134
|
+
description='Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.',
|
135
|
+
examples=['source_railz.components.MyCustomPaginationStrategy'],
|
136
|
+
title='Class Name',
|
137
137
|
)
|
138
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
138
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
139
139
|
|
140
140
|
|
141
141
|
class CustomRecordExtractor(BaseModel):
|
142
142
|
class Config:
|
143
143
|
extra = Extra.allow
|
144
144
|
|
145
|
-
type: Literal[
|
145
|
+
type: Literal['CustomRecordExtractor']
|
146
146
|
class_name: str = Field(
|
147
147
|
...,
|
148
|
-
description=
|
149
|
-
examples=[
|
150
|
-
title=
|
148
|
+
description='Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.',
|
149
|
+
examples=['source_railz.components.MyCustomRecordExtractor'],
|
150
|
+
title='Class Name',
|
151
151
|
)
|
152
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
152
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
153
153
|
|
154
154
|
|
155
155
|
class CustomRecordFilter(BaseModel):
|
@@ -159,8 +159,8 @@ class CustomRecordFilter(BaseModel):
|
|
159
159
|
type: Literal['CustomRecordFilter']
|
160
160
|
class_name: str = Field(
|
161
161
|
...,
|
162
|
-
description='Fully-qualified name of the class that will be implementing the custom record
|
163
|
-
examples=['source_railz.components.
|
162
|
+
description='Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_<name>.<package>.<class_name>`.',
|
163
|
+
examples=['source_railz.components.MyCustomCustomRecordFilter'],
|
164
164
|
title='Class Name',
|
165
165
|
)
|
166
166
|
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
@@ -170,465 +170,471 @@ class CustomRequester(BaseModel):
|
|
170
170
|
class Config:
|
171
171
|
extra = Extra.allow
|
172
172
|
|
173
|
-
type: Literal[
|
173
|
+
type: Literal['CustomRequester']
|
174
174
|
class_name: str = Field(
|
175
175
|
...,
|
176
|
-
description=
|
177
|
-
examples=[
|
178
|
-
title=
|
176
|
+
description='Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.',
|
177
|
+
examples=['source_railz.components.MyCustomRecordExtractor'],
|
178
|
+
title='Class Name',
|
179
179
|
)
|
180
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
180
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
181
181
|
|
182
182
|
|
183
183
|
class CustomRetriever(BaseModel):
|
184
184
|
class Config:
|
185
185
|
extra = Extra.allow
|
186
186
|
|
187
|
-
type: Literal[
|
187
|
+
type: Literal['CustomRetriever']
|
188
188
|
class_name: str = Field(
|
189
189
|
...,
|
190
|
-
description=
|
191
|
-
examples=[
|
192
|
-
title=
|
190
|
+
description='Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.',
|
191
|
+
examples=['source_railz.components.MyCustomRetriever'],
|
192
|
+
title='Class Name',
|
193
193
|
)
|
194
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
194
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
195
195
|
|
196
196
|
|
197
197
|
class CustomPartitionRouter(BaseModel):
|
198
198
|
class Config:
|
199
199
|
extra = Extra.allow
|
200
200
|
|
201
|
-
type: Literal[
|
201
|
+
type: Literal['CustomPartitionRouter']
|
202
202
|
class_name: str = Field(
|
203
203
|
...,
|
204
|
-
description=
|
205
|
-
examples=[
|
206
|
-
title=
|
204
|
+
description='Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.',
|
205
|
+
examples=['source_railz.components.MyCustomPartitionRouter'],
|
206
|
+
title='Class Name',
|
207
207
|
)
|
208
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
208
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
209
209
|
|
210
210
|
|
211
211
|
class CustomTransformation(BaseModel):
|
212
212
|
class Config:
|
213
213
|
extra = Extra.allow
|
214
214
|
|
215
|
-
type: Literal[
|
215
|
+
type: Literal['CustomTransformation']
|
216
216
|
class_name: str = Field(
|
217
217
|
...,
|
218
|
-
description=
|
219
|
-
examples=[
|
220
|
-
title=
|
218
|
+
description='Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.',
|
219
|
+
examples=['source_railz.components.MyCustomTransformation'],
|
220
|
+
title='Class Name',
|
221
221
|
)
|
222
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
222
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
223
223
|
|
224
224
|
|
225
225
|
class RefreshTokenUpdater(BaseModel):
|
226
226
|
refresh_token_name: Optional[str] = Field(
|
227
|
-
|
228
|
-
description=
|
229
|
-
examples=[
|
230
|
-
title=
|
227
|
+
'refresh_token',
|
228
|
+
description='The name of the property which contains the updated refresh token in the response from the token refresh endpoint.',
|
229
|
+
examples=['refresh_token'],
|
230
|
+
title='Refresh Token Property Name',
|
231
231
|
)
|
232
232
|
access_token_config_path: Optional[List[str]] = Field(
|
233
|
-
[
|
234
|
-
description=
|
235
|
-
examples=[[
|
236
|
-
title=
|
233
|
+
['credentials', 'access_token'],
|
234
|
+
description='Config path to the access token. Make sure the field actually exists in the config.',
|
235
|
+
examples=[['credentials', 'access_token'], ['access_token']],
|
236
|
+
title='Config Path To Access Token',
|
237
237
|
)
|
238
238
|
refresh_token_config_path: Optional[List[str]] = Field(
|
239
|
-
[
|
240
|
-
description=
|
241
|
-
examples=[[
|
242
|
-
title=
|
239
|
+
['credentials', 'refresh_token'],
|
240
|
+
description='Config path to the access token. Make sure the field actually exists in the config.',
|
241
|
+
examples=[['credentials', 'refresh_token'], ['refresh_token']],
|
242
|
+
title='Config Path To Refresh Token',
|
243
243
|
)
|
244
244
|
token_expiry_date_config_path: Optional[List[str]] = Field(
|
245
|
-
[
|
246
|
-
description=
|
247
|
-
examples=[[
|
248
|
-
title=
|
245
|
+
['credentials', 'token_expiry_date'],
|
246
|
+
description='Config path to the expiry date. Make sure actually exists in the config.',
|
247
|
+
examples=[['credentials', 'token_expiry_date']],
|
248
|
+
title='Config Path To Expiry Date',
|
249
249
|
)
|
250
250
|
|
251
251
|
|
252
252
|
class OAuthAuthenticator(BaseModel):
|
253
|
-
type: Literal[
|
253
|
+
type: Literal['OAuthAuthenticator']
|
254
254
|
client_id: str = Field(
|
255
255
|
...,
|
256
|
-
description=
|
256
|
+
description='The OAuth client ID. Fill it in the user inputs.',
|
257
257
|
examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"],
|
258
|
-
title=
|
258
|
+
title='Client ID',
|
259
259
|
)
|
260
260
|
client_secret: str = Field(
|
261
261
|
...,
|
262
|
-
description=
|
262
|
+
description='The OAuth client secret. Fill it in the user inputs.',
|
263
263
|
examples=[
|
264
264
|
"{{ config['client_secret }}",
|
265
265
|
"{{ config['credentials']['client_secret }}",
|
266
266
|
],
|
267
|
-
title=
|
267
|
+
title='Client Secret',
|
268
268
|
)
|
269
269
|
refresh_token: Optional[str] = Field(
|
270
270
|
None,
|
271
|
-
description=
|
271
|
+
description='Credential artifact used to get a new access token.',
|
272
272
|
examples=[
|
273
273
|
"{{ config['refresh_token'] }}",
|
274
274
|
"{{ config['credentials]['refresh_token'] }}",
|
275
275
|
],
|
276
|
-
title=
|
276
|
+
title='Refresh Token',
|
277
277
|
)
|
278
278
|
token_refresh_endpoint: str = Field(
|
279
279
|
...,
|
280
|
-
description=
|
281
|
-
examples=[
|
282
|
-
title=
|
280
|
+
description='The full URL to call to obtain a new access token.',
|
281
|
+
examples=['https://connect.squareup.com/oauth2/token'],
|
282
|
+
title='Token Refresh Endpoint',
|
283
283
|
)
|
284
284
|
access_token_name: Optional[str] = Field(
|
285
|
-
|
286
|
-
description=
|
287
|
-
examples=[
|
288
|
-
title=
|
285
|
+
'access_token',
|
286
|
+
description='The name of the property which contains the access token in the response from the token refresh endpoint.',
|
287
|
+
examples=['access_token'],
|
288
|
+
title='Access Token Property Name',
|
289
289
|
)
|
290
290
|
expires_in_name: Optional[str] = Field(
|
291
|
-
|
292
|
-
description=
|
293
|
-
examples=[
|
294
|
-
title=
|
291
|
+
'expires_in',
|
292
|
+
description='The name of the property which contains the expiry date in the response from the token refresh endpoint.',
|
293
|
+
examples=['expires_in'],
|
294
|
+
title='Token Expiry Property Name',
|
295
295
|
)
|
296
296
|
grant_type: Optional[str] = Field(
|
297
|
-
|
298
|
-
description=
|
299
|
-
examples=[
|
300
|
-
title=
|
297
|
+
'refresh_token',
|
298
|
+
description='Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.',
|
299
|
+
examples=['refresh_token', 'client_credentials'],
|
300
|
+
title='Grant Type',
|
301
301
|
)
|
302
302
|
refresh_request_body: Optional[Dict[str, Any]] = Field(
|
303
303
|
None,
|
304
|
-
description=
|
304
|
+
description='Body of the request sent to get a new access token.',
|
305
305
|
examples=[
|
306
306
|
{
|
307
|
-
|
308
|
-
|
309
|
-
|
307
|
+
'applicationId': "{{ config['application_id'] }}",
|
308
|
+
'applicationSecret': "{{ config['application_secret'] }}",
|
309
|
+
'token': "{{ config['token'] }}",
|
310
310
|
}
|
311
311
|
],
|
312
|
-
title=
|
312
|
+
title='Refresh Request Body',
|
313
313
|
)
|
314
314
|
scopes: Optional[List[str]] = Field(
|
315
315
|
None,
|
316
|
-
description=
|
317
|
-
examples=[
|
318
|
-
|
316
|
+
description='List of scopes that should be granted to the access token.',
|
317
|
+
examples=[
|
318
|
+
['crm.list.read', 'crm.objects.contacts.read', 'crm.schema.contacts.read']
|
319
|
+
],
|
320
|
+
title='Scopes',
|
319
321
|
)
|
320
322
|
token_expiry_date: Optional[str] = Field(
|
321
323
|
None,
|
322
|
-
description=
|
323
|
-
examples=[
|
324
|
-
title=
|
324
|
+
description='The access token expiry date.',
|
325
|
+
examples=['2023-04-06T07:12:10.421833+00:00', 1680842386],
|
326
|
+
title='Token Expiry Date',
|
325
327
|
)
|
326
328
|
token_expiry_date_format: Optional[str] = Field(
|
327
329
|
None,
|
328
|
-
description=
|
329
|
-
examples=[
|
330
|
-
title=
|
330
|
+
description='The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.',
|
331
|
+
examples=['%Y-%m-%d %H:%M:%S.%f+00:00'],
|
332
|
+
title='Token Expiry Date Format',
|
331
333
|
)
|
332
334
|
refresh_token_updater: Optional[RefreshTokenUpdater] = Field(
|
333
335
|
None,
|
334
|
-
description=
|
335
|
-
title=
|
336
|
+
description='When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.',
|
337
|
+
title='Token Updater',
|
336
338
|
)
|
337
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
339
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
338
340
|
|
339
341
|
|
340
342
|
class ExponentialBackoffStrategy(BaseModel):
|
341
|
-
type: Literal[
|
343
|
+
type: Literal['ExponentialBackoffStrategy']
|
342
344
|
factor: Optional[Union[float, str]] = Field(
|
343
345
|
5,
|
344
|
-
description=
|
345
|
-
examples=[5, 5.5,
|
346
|
-
title=
|
346
|
+
description='Multiplicative constant applied on each retry.',
|
347
|
+
examples=[5, 5.5, '10'],
|
348
|
+
title='Factor',
|
347
349
|
)
|
348
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
350
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
349
351
|
|
350
352
|
|
351
353
|
class SessionTokenRequestBearerAuthenticator(BaseModel):
|
352
|
-
type: Literal[
|
354
|
+
type: Literal['Bearer']
|
353
355
|
|
354
356
|
|
355
357
|
class HttpMethod(Enum):
|
356
|
-
GET =
|
357
|
-
POST =
|
358
|
+
GET = 'GET'
|
359
|
+
POST = 'POST'
|
358
360
|
|
359
361
|
|
360
362
|
class Action(Enum):
|
361
|
-
SUCCESS =
|
362
|
-
FAIL =
|
363
|
-
RETRY =
|
364
|
-
IGNORE =
|
363
|
+
SUCCESS = 'SUCCESS'
|
364
|
+
FAIL = 'FAIL'
|
365
|
+
RETRY = 'RETRY'
|
366
|
+
IGNORE = 'IGNORE'
|
365
367
|
|
366
368
|
|
367
369
|
class HttpResponseFilter(BaseModel):
|
368
|
-
type: Literal[
|
370
|
+
type: Literal['HttpResponseFilter']
|
369
371
|
action: Action = Field(
|
370
372
|
...,
|
371
|
-
description=
|
372
|
-
examples=[
|
373
|
-
title=
|
373
|
+
description='Action to execute if a response matches the filter.',
|
374
|
+
examples=['SUCCESS', 'FAIL', 'RETRY', 'IGNORE'],
|
375
|
+
title='Action',
|
374
376
|
)
|
375
377
|
error_message: Optional[str] = Field(
|
376
378
|
None,
|
377
|
-
description=
|
378
|
-
title=
|
379
|
+
description='Error Message to display if the response matches the filter.',
|
380
|
+
title='Error Message',
|
379
381
|
)
|
380
382
|
error_message_contains: Optional[str] = Field(
|
381
383
|
None,
|
382
|
-
description=
|
383
|
-
example=[
|
384
|
-
title=
|
384
|
+
description='Match the response if its error message contains the substring.',
|
385
|
+
example=['This API operation is not enabled for this site'],
|
386
|
+
title='Error Message Substring',
|
385
387
|
)
|
386
388
|
http_codes: Optional[List[int]] = Field(
|
387
389
|
None,
|
388
|
-
description=
|
390
|
+
description='Match the response if its HTTP code is included in this list.',
|
389
391
|
examples=[[420, 429], [500]],
|
390
|
-
title=
|
392
|
+
title='HTTP Codes',
|
391
393
|
)
|
392
394
|
predicate: Optional[str] = Field(
|
393
395
|
None,
|
394
|
-
description=
|
396
|
+
description='Match the response if the predicate evaluates to true.',
|
395
397
|
examples=[
|
396
398
|
"{{ 'Too much requests' in response }}",
|
397
399
|
"{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}",
|
398
400
|
],
|
399
|
-
title=
|
401
|
+
title='Predicate',
|
400
402
|
)
|
401
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
403
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
402
404
|
|
403
405
|
|
404
406
|
class InlineSchemaLoader(BaseModel):
|
405
|
-
type: Literal[
|
407
|
+
type: Literal['InlineSchemaLoader']
|
406
408
|
schema_: Optional[Dict[str, Any]] = Field(
|
407
409
|
None,
|
408
|
-
alias=
|
410
|
+
alias='schema',
|
409
411
|
description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.',
|
410
|
-
title=
|
412
|
+
title='Schema',
|
411
413
|
)
|
412
414
|
|
413
415
|
|
414
416
|
class JsonFileSchemaLoader(BaseModel):
|
415
|
-
type: Literal[
|
417
|
+
type: Literal['JsonFileSchemaLoader']
|
416
418
|
file_path: Optional[str] = Field(
|
417
419
|
None,
|
418
420
|
description="Path to the JSON file defining the schema. The path is relative to the connector module's root.",
|
419
|
-
example=[
|
420
|
-
title=
|
421
|
+
example=['./schemas/users.json'],
|
422
|
+
title='File Path',
|
421
423
|
)
|
422
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
424
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
423
425
|
|
424
426
|
|
425
427
|
class JsonDecoder(BaseModel):
|
426
|
-
type: Literal[
|
428
|
+
type: Literal['JsonDecoder']
|
427
429
|
|
428
430
|
|
429
431
|
class MinMaxDatetime(BaseModel):
|
430
|
-
type: Literal[
|
432
|
+
type: Literal['MinMaxDatetime']
|
431
433
|
datetime: str = Field(
|
432
434
|
...,
|
433
|
-
description=
|
434
|
-
examples=[
|
435
|
-
title=
|
435
|
+
description='Datetime value.',
|
436
|
+
examples=['2021-01-01', '2021-01-01T00:00:00Z', "{{ config['start_time'] }}"],
|
437
|
+
title='Datetime',
|
436
438
|
)
|
437
439
|
datetime_format: Optional[str] = Field(
|
438
|
-
|
440
|
+
'',
|
439
441
|
description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
|
440
|
-
examples=[
|
441
|
-
title=
|
442
|
+
examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s'],
|
443
|
+
title='Datetime Format',
|
442
444
|
)
|
443
445
|
max_datetime: Optional[str] = Field(
|
444
446
|
None,
|
445
|
-
description=
|
446
|
-
examples=[
|
447
|
-
title=
|
447
|
+
description='Ceiling applied on the datetime value. Must be formatted with the datetime_format field.',
|
448
|
+
examples=['2021-01-01T00:00:00Z', '2021-01-01'],
|
449
|
+
title='Max Datetime',
|
448
450
|
)
|
449
451
|
min_datetime: Optional[str] = Field(
|
450
452
|
None,
|
451
|
-
description=
|
452
|
-
examples=[
|
453
|
-
title=
|
453
|
+
description='Floor applied on the datetime value. Must be formatted with the datetime_format field.',
|
454
|
+
examples=['2010-01-01T00:00:00Z', '2010-01-01'],
|
455
|
+
title='Min Datetime',
|
454
456
|
)
|
455
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
457
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
456
458
|
|
457
459
|
|
458
460
|
class NoAuth(BaseModel):
|
459
|
-
type: Literal[
|
460
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
461
|
+
type: Literal['NoAuth']
|
462
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
461
463
|
|
462
464
|
|
463
465
|
class NoPagination(BaseModel):
|
464
|
-
type: Literal[
|
466
|
+
type: Literal['NoPagination']
|
465
467
|
|
466
468
|
|
467
469
|
class OAuthConfigSpecification(BaseModel):
|
468
470
|
class Config:
|
469
471
|
extra = Extra.allow
|
470
472
|
|
471
|
-
oauth_user_input_from_connector_config_specification: Optional[
|
473
|
+
oauth_user_input_from_connector_config_specification: Optional[
|
474
|
+
Dict[str, Any]
|
475
|
+
] = Field(
|
472
476
|
None,
|
473
477
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }",
|
474
478
|
examples=[
|
475
|
-
{
|
479
|
+
{'app_id': {'type': 'string', 'path_in_connector_config': ['app_id']}},
|
476
480
|
{
|
477
|
-
|
478
|
-
|
479
|
-
|
481
|
+
'app_id': {
|
482
|
+
'type': 'string',
|
483
|
+
'path_in_connector_config': ['info', 'app_id'],
|
480
484
|
}
|
481
485
|
},
|
482
486
|
],
|
483
|
-
title=
|
487
|
+
title='OAuth user input',
|
484
488
|
)
|
485
489
|
complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
|
486
490
|
None,
|
487
491
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
|
488
492
|
examples=[
|
489
493
|
{
|
490
|
-
|
491
|
-
|
492
|
-
|
494
|
+
'refresh_token': {
|
495
|
+
'type': 'string,',
|
496
|
+
'path_in_connector_config': ['credentials', 'refresh_token'],
|
493
497
|
}
|
494
498
|
}
|
495
499
|
],
|
496
|
-
title=
|
500
|
+
title='OAuth output specification',
|
497
501
|
)
|
498
502
|
complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field(
|
499
503
|
None,
|
500
|
-
description=
|
501
|
-
examples=[
|
502
|
-
|
504
|
+
description='OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }',
|
505
|
+
examples=[
|
506
|
+
{'client_id': {'type': 'string'}, 'client_secret': {'type': 'string'}}
|
507
|
+
],
|
508
|
+
title='OAuth input specification',
|
503
509
|
)
|
504
510
|
complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field(
|
505
511
|
None,
|
506
512
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }",
|
507
513
|
examples=[
|
508
514
|
{
|
509
|
-
|
510
|
-
|
511
|
-
|
515
|
+
'client_id': {
|
516
|
+
'type': 'string,',
|
517
|
+
'path_in_connector_config': ['credentials', 'client_id'],
|
512
518
|
},
|
513
|
-
|
514
|
-
|
515
|
-
|
519
|
+
'client_secret': {
|
520
|
+
'type': 'string,',
|
521
|
+
'path_in_connector_config': ['credentials', 'client_secret'],
|
516
522
|
},
|
517
523
|
}
|
518
524
|
],
|
519
|
-
title=
|
525
|
+
title='OAuth server output specification',
|
520
526
|
)
|
521
527
|
|
522
528
|
|
523
529
|
class OffsetIncrement(BaseModel):
|
524
|
-
type: Literal[
|
530
|
+
type: Literal['OffsetIncrement']
|
525
531
|
page_size: Optional[Union[int, str]] = Field(
|
526
532
|
None,
|
527
|
-
description=
|
533
|
+
description='The number of records to include in each pages.',
|
528
534
|
examples=[100, "{{ config['page_size'] }}"],
|
529
|
-
title=
|
535
|
+
title='Limit',
|
530
536
|
)
|
531
537
|
inject_on_first_request: Optional[bool] = Field(
|
532
538
|
False,
|
533
|
-
description=
|
534
|
-
title=
|
539
|
+
description='Using the `offset` with value `0` during the first request',
|
540
|
+
title='Inject Offset',
|
535
541
|
)
|
536
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
542
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
537
543
|
|
538
544
|
|
539
545
|
class PageIncrement(BaseModel):
|
540
|
-
type: Literal[
|
546
|
+
type: Literal['PageIncrement']
|
541
547
|
page_size: Optional[int] = Field(
|
542
548
|
None,
|
543
|
-
description=
|
544
|
-
examples=[100,
|
545
|
-
title=
|
549
|
+
description='The number of records to include in each pages.',
|
550
|
+
examples=[100, '100'],
|
551
|
+
title='Page Size',
|
546
552
|
)
|
547
553
|
start_from_page: Optional[int] = Field(
|
548
554
|
0,
|
549
|
-
description=
|
555
|
+
description='Index of the first page to request.',
|
550
556
|
examples=[0, 1],
|
551
|
-
title=
|
557
|
+
title='Start From Page',
|
552
558
|
)
|
553
559
|
inject_on_first_request: Optional[bool] = Field(
|
554
560
|
False,
|
555
|
-
description=
|
556
|
-
title=
|
561
|
+
description='Using the `page number` with value defined by `start_from_page` during the first request',
|
562
|
+
title='Inject Page Number',
|
557
563
|
)
|
558
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
564
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
559
565
|
|
560
566
|
|
561
567
|
class PrimaryKey(BaseModel):
|
562
568
|
__root__: Union[str, List[str], List[List[str]]] = Field(
|
563
569
|
...,
|
564
|
-
description=
|
565
|
-
examples=[
|
566
|
-
title=
|
570
|
+
description='The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.',
|
571
|
+
examples=['id', ['code', 'type']],
|
572
|
+
title='Primary Key',
|
567
573
|
)
|
568
574
|
|
569
575
|
|
570
576
|
class RecordFilter(BaseModel):
|
571
|
-
type: Literal[
|
577
|
+
type: Literal['RecordFilter']
|
572
578
|
condition: Optional[str] = Field(
|
573
|
-
|
574
|
-
description=
|
579
|
+
'',
|
580
|
+
description='The predicate to filter a record. Records will be removed if evaluated to False.',
|
575
581
|
examples=[
|
576
582
|
"{{ record['created_at'] >= stream_interval['start_time'] }}",
|
577
583
|
"{{ record.status in ['active', 'expired'] }}",
|
578
584
|
],
|
579
585
|
)
|
580
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
586
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
581
587
|
|
582
588
|
|
583
589
|
class SchemaNormalization(Enum):
|
584
|
-
None_ =
|
585
|
-
Default =
|
590
|
+
None_ = 'None'
|
591
|
+
Default = 'Default'
|
586
592
|
|
587
593
|
|
588
594
|
class RemoveFields(BaseModel):
|
589
|
-
type: Literal[
|
595
|
+
type: Literal['RemoveFields']
|
590
596
|
condition: Optional[str] = Field(
|
591
|
-
|
592
|
-
description=
|
597
|
+
'',
|
598
|
+
description='The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,',
|
593
599
|
examples=[
|
594
600
|
"{{ property|string == '' }}",
|
595
|
-
|
596
|
-
|
601
|
+
'{{ property is integer }}',
|
602
|
+
'{{ property|length > 5 }}',
|
597
603
|
"{{ property == 'some_string_to_match' }}",
|
598
604
|
],
|
599
605
|
)
|
600
606
|
field_pointers: List[List[str]] = Field(
|
601
607
|
...,
|
602
|
-
description=
|
603
|
-
examples=[[
|
604
|
-
title=
|
608
|
+
description='Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.',
|
609
|
+
examples=[['tags'], [['content', 'html'], ['content', 'plain_text']]],
|
610
|
+
title='Field Paths',
|
605
611
|
)
|
606
612
|
|
607
613
|
|
608
614
|
class RequestPath(BaseModel):
|
609
|
-
type: Literal[
|
615
|
+
type: Literal['RequestPath']
|
610
616
|
|
611
617
|
|
612
618
|
class InjectInto(Enum):
|
613
|
-
request_parameter =
|
614
|
-
header =
|
615
|
-
body_data =
|
616
|
-
body_json =
|
619
|
+
request_parameter = 'request_parameter'
|
620
|
+
header = 'header'
|
621
|
+
body_data = 'body_data'
|
622
|
+
body_json = 'body_json'
|
617
623
|
|
618
624
|
|
619
625
|
class RequestOption(BaseModel):
|
620
|
-
type: Literal[
|
626
|
+
type: Literal['RequestOption']
|
621
627
|
field_name: str = Field(
|
622
628
|
...,
|
623
|
-
description=
|
624
|
-
examples=[
|
625
|
-
title=
|
629
|
+
description='Configures which key should be used in the location that the descriptor is being injected into',
|
630
|
+
examples=['segment_id'],
|
631
|
+
title='Request Option',
|
626
632
|
)
|
627
633
|
inject_into: InjectInto = Field(
|
628
634
|
...,
|
629
|
-
description=
|
630
|
-
examples=[
|
631
|
-
title=
|
635
|
+
description='Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.',
|
636
|
+
examples=['request_parameter', 'header', 'body_data', 'body_json'],
|
637
|
+
title='Inject Into',
|
632
638
|
)
|
633
639
|
|
634
640
|
|
@@ -640,106 +646,106 @@ class Schemas(BaseModel):
|
|
640
646
|
|
641
647
|
|
642
648
|
class LegacySessionTokenAuthenticator(BaseModel):
|
643
|
-
type: Literal[
|
649
|
+
type: Literal['LegacySessionTokenAuthenticator']
|
644
650
|
header: str = Field(
|
645
651
|
...,
|
646
|
-
description=
|
647
|
-
examples=[
|
648
|
-
title=
|
652
|
+
description='The name of the session token header that will be injected in the request',
|
653
|
+
examples=['X-Session'],
|
654
|
+
title='Session Request Header',
|
649
655
|
)
|
650
656
|
login_url: str = Field(
|
651
657
|
...,
|
652
|
-
description=
|
653
|
-
examples=[
|
654
|
-
title=
|
658
|
+
description='Path of the login URL (do not include the base URL)',
|
659
|
+
examples=['session'],
|
660
|
+
title='Login Path',
|
655
661
|
)
|
656
662
|
session_token: Optional[str] = Field(
|
657
663
|
None,
|
658
|
-
description=
|
664
|
+
description='Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair',
|
659
665
|
example=["{{ config['session_token'] }}"],
|
660
|
-
title=
|
666
|
+
title='Session Token',
|
661
667
|
)
|
662
668
|
session_token_response_key: str = Field(
|
663
669
|
...,
|
664
|
-
description=
|
665
|
-
examples=[
|
666
|
-
title=
|
670
|
+
description='Name of the key of the session token to be extracted from the response',
|
671
|
+
examples=['id'],
|
672
|
+
title='Response Token Response Key',
|
667
673
|
)
|
668
674
|
username: Optional[str] = Field(
|
669
675
|
None,
|
670
|
-
description=
|
676
|
+
description='Username used to authenticate and obtain a session token',
|
671
677
|
examples=[" {{ config['username'] }}"],
|
672
|
-
title=
|
678
|
+
title='Username',
|
673
679
|
)
|
674
680
|
password: Optional[str] = Field(
|
675
|
-
|
676
|
-
description=
|
677
|
-
examples=["{{ config['password'] }}",
|
678
|
-
title=
|
681
|
+
'',
|
682
|
+
description='Password used to authenticate and obtain a session token',
|
683
|
+
examples=["{{ config['password'] }}", ''],
|
684
|
+
title='Password',
|
679
685
|
)
|
680
686
|
validate_session_url: str = Field(
|
681
687
|
...,
|
682
|
-
description=
|
683
|
-
examples=[
|
684
|
-
title=
|
688
|
+
description='Path of the URL to use to validate that the session token is valid (do not include the base URL)',
|
689
|
+
examples=['user/current'],
|
690
|
+
title='Validate Session Path',
|
685
691
|
)
|
686
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
692
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
687
693
|
|
688
694
|
|
689
695
|
class ValueType(Enum):
|
690
|
-
string =
|
691
|
-
number =
|
692
|
-
integer =
|
693
|
-
boolean =
|
696
|
+
string = 'string'
|
697
|
+
number = 'number'
|
698
|
+
integer = 'integer'
|
699
|
+
boolean = 'boolean'
|
694
700
|
|
695
701
|
|
696
702
|
class WaitTimeFromHeader(BaseModel):
|
697
|
-
type: Literal[
|
703
|
+
type: Literal['WaitTimeFromHeader']
|
698
704
|
header: str = Field(
|
699
705
|
...,
|
700
|
-
description=
|
701
|
-
examples=[
|
702
|
-
title=
|
706
|
+
description='The name of the response header defining how long to wait before retrying.',
|
707
|
+
examples=['Retry-After'],
|
708
|
+
title='Response Header Name',
|
703
709
|
)
|
704
710
|
regex: Optional[str] = Field(
|
705
711
|
None,
|
706
|
-
description=
|
707
|
-
examples=[
|
708
|
-
title=
|
712
|
+
description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.',
|
713
|
+
examples=['([-+]?\\d+)'],
|
714
|
+
title='Extraction Regex',
|
709
715
|
)
|
710
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
716
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
711
717
|
|
712
718
|
|
713
719
|
class WaitUntilTimeFromHeader(BaseModel):
|
714
|
-
type: Literal[
|
720
|
+
type: Literal['WaitUntilTimeFromHeader']
|
715
721
|
header: str = Field(
|
716
722
|
...,
|
717
|
-
description=
|
718
|
-
examples=[
|
719
|
-
title=
|
723
|
+
description='The name of the response header defining how long to wait before retrying.',
|
724
|
+
examples=['wait_time'],
|
725
|
+
title='Response Header',
|
720
726
|
)
|
721
727
|
min_wait: Optional[Union[float, str]] = Field(
|
722
728
|
None,
|
723
|
-
description=
|
724
|
-
examples=[10,
|
725
|
-
title=
|
729
|
+
description='Minimum time to wait before retrying.',
|
730
|
+
examples=[10, '60'],
|
731
|
+
title='Minimum Wait Time',
|
726
732
|
)
|
727
733
|
regex: Optional[str] = Field(
|
728
734
|
None,
|
729
|
-
description=
|
730
|
-
examples=[
|
731
|
-
title=
|
735
|
+
description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.',
|
736
|
+
examples=['([-+]?\\d+)'],
|
737
|
+
title='Extraction Regex',
|
732
738
|
)
|
733
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
739
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
734
740
|
|
735
741
|
|
736
742
|
class AddedFieldDefinition(BaseModel):
|
737
|
-
type: Literal[
|
743
|
+
type: Literal['AddedFieldDefinition']
|
738
744
|
path: List[str] = Field(
|
739
745
|
...,
|
740
|
-
description=
|
741
|
-
examples=[[
|
742
|
-
title=
|
746
|
+
description='List of strings defining the path where to add the value on the record.',
|
747
|
+
examples=[['segment_id'], ['metadata', 'segment_id']],
|
748
|
+
title='Path',
|
743
749
|
)
|
744
750
|
value: str = Field(
|
745
751
|
...,
|
@@ -749,185 +755,187 @@ class AddedFieldDefinition(BaseModel):
|
|
749
755
|
"{{ record['MetaData']['LastUpdatedTime'] }}",
|
750
756
|
"{{ stream_partition['segment_id'] }}",
|
751
757
|
],
|
752
|
-
title=
|
758
|
+
title='Value',
|
753
759
|
)
|
754
760
|
value_type: Optional[ValueType] = Field(
|
755
761
|
None,
|
756
|
-
description=
|
757
|
-
title=
|
762
|
+
description='Type of the value. If not specified, the type will be inferred from the value.',
|
763
|
+
title='Value Type',
|
758
764
|
)
|
759
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
765
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
760
766
|
|
761
767
|
|
762
768
|
class AddFields(BaseModel):
|
763
|
-
type: Literal[
|
769
|
+
type: Literal['AddFields']
|
764
770
|
fields: List[AddedFieldDefinition] = Field(
|
765
771
|
...,
|
766
|
-
description=
|
767
|
-
title=
|
772
|
+
description='List of transformations (path and corresponding value) that will be added to the record.',
|
773
|
+
title='Fields',
|
768
774
|
)
|
769
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
775
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
770
776
|
|
771
777
|
|
772
778
|
class ApiKeyAuthenticator(BaseModel):
|
773
|
-
type: Literal[
|
779
|
+
type: Literal['ApiKeyAuthenticator']
|
774
780
|
api_token: Optional[str] = Field(
|
775
781
|
None,
|
776
|
-
description=
|
782
|
+
description='The API key to inject in the request. Fill it in the user inputs.',
|
777
783
|
examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"],
|
778
|
-
title=
|
784
|
+
title='API Key',
|
779
785
|
)
|
780
786
|
header: Optional[str] = Field(
|
781
787
|
None,
|
782
|
-
description=
|
783
|
-
examples=[
|
784
|
-
title=
|
788
|
+
description='The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.',
|
789
|
+
examples=['Authorization', 'Api-Token', 'X-Auth-Token'],
|
790
|
+
title='Header Name',
|
785
791
|
)
|
786
792
|
inject_into: Optional[RequestOption] = Field(
|
787
793
|
None,
|
788
|
-
description=
|
794
|
+
description='Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.',
|
789
795
|
examples=[
|
790
|
-
{
|
791
|
-
{
|
796
|
+
{'inject_into': 'header', 'field_name': 'Authorization'},
|
797
|
+
{'inject_into': 'request_parameter', 'field_name': 'authKey'},
|
792
798
|
],
|
793
|
-
title=
|
799
|
+
title='Inject API Key Into Outgoing HTTP Request',
|
794
800
|
)
|
795
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
801
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
796
802
|
|
797
803
|
|
798
804
|
class AuthFlow(BaseModel):
|
799
|
-
auth_flow_type: Optional[AuthFlowType] = Field(
|
805
|
+
auth_flow_type: Optional[AuthFlowType] = Field(
|
806
|
+
None, description='The type of auth to use', title='Auth flow type'
|
807
|
+
)
|
800
808
|
predicate_key: Optional[List[str]] = Field(
|
801
809
|
None,
|
802
|
-
description=
|
803
|
-
examples=[[
|
804
|
-
title=
|
810
|
+
description='JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.',
|
811
|
+
examples=[['credentials', 'auth_type']],
|
812
|
+
title='Predicate key',
|
805
813
|
)
|
806
814
|
predicate_value: Optional[str] = Field(
|
807
815
|
None,
|
808
|
-
description=
|
809
|
-
examples=[
|
810
|
-
title=
|
816
|
+
description='Value of the predicate_key fields for the advanced auth to be applicable.',
|
817
|
+
examples=['Oauth'],
|
818
|
+
title='Predicate value',
|
811
819
|
)
|
812
820
|
oauth_config_specification: Optional[OAuthConfigSpecification] = None
|
813
821
|
|
814
822
|
|
815
823
|
class CursorPagination(BaseModel):
|
816
|
-
type: Literal[
|
824
|
+
type: Literal['CursorPagination']
|
817
825
|
cursor_value: str = Field(
|
818
826
|
...,
|
819
|
-
description=
|
827
|
+
description='Value of the cursor defining the next page to fetch.',
|
820
828
|
examples=[
|
821
|
-
|
829
|
+
'{{ headers.link.next.cursor }}',
|
822
830
|
"{{ last_records[-1]['key'] }}",
|
823
831
|
"{{ response['nextPage'] }}",
|
824
832
|
],
|
825
|
-
title=
|
833
|
+
title='Cursor Value',
|
826
834
|
)
|
827
835
|
page_size: Optional[int] = Field(
|
828
836
|
None,
|
829
|
-
description=
|
837
|
+
description='The number of records to include in each pages.',
|
830
838
|
examples=[100],
|
831
|
-
title=
|
839
|
+
title='Page Size',
|
832
840
|
)
|
833
841
|
stop_condition: Optional[str] = Field(
|
834
842
|
None,
|
835
|
-
description=
|
843
|
+
description='Template string evaluating when to stop paginating.',
|
836
844
|
examples=[
|
837
|
-
|
845
|
+
'{{ response.data.has_more is false }}',
|
838
846
|
"{{ 'next' not in headers['link'] }}",
|
839
847
|
],
|
840
|
-
title=
|
848
|
+
title='Stop Condition',
|
841
849
|
)
|
842
850
|
decoder: Optional[JsonDecoder] = Field(
|
843
851
|
None,
|
844
|
-
description=
|
845
|
-
title=
|
852
|
+
description='Component decoding the response so records can be extracted.',
|
853
|
+
title='Decoder',
|
846
854
|
)
|
847
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
855
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
848
856
|
|
849
857
|
|
850
858
|
class DatetimeBasedCursor(BaseModel):
|
851
|
-
type: Literal[
|
859
|
+
type: Literal['DatetimeBasedCursor']
|
852
860
|
cursor_field: str = Field(
|
853
861
|
...,
|
854
|
-
description=
|
855
|
-
examples=[
|
856
|
-
title=
|
862
|
+
description='The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.',
|
863
|
+
examples=['created_at', "{{ config['record_cursor'] }}"],
|
864
|
+
title='Cursor Field',
|
857
865
|
)
|
858
866
|
datetime_format: str = Field(
|
859
867
|
...,
|
860
|
-
description=
|
861
|
-
examples=[
|
862
|
-
title=
|
868
|
+
description='The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
|
869
|
+
examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s', '%ms'],
|
870
|
+
title='Outgoing Datetime Format',
|
863
871
|
)
|
864
872
|
start_datetime: Union[str, MinMaxDatetime] = Field(
|
865
873
|
...,
|
866
|
-
description=
|
867
|
-
examples=[
|
868
|
-
title=
|
874
|
+
description='The datetime that determines the earliest record that should be synced.',
|
875
|
+
examples=['2020-01-1T00:00:00Z', "{{ config['start_time'] }}"],
|
876
|
+
title='Start Datetime',
|
869
877
|
)
|
870
878
|
cursor_datetime_formats: Optional[List[str]] = Field(
|
871
879
|
None,
|
872
|
-
description=
|
873
|
-
title=
|
880
|
+
description='The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.',
|
881
|
+
title='Cursor Datetime Formats',
|
874
882
|
)
|
875
883
|
cursor_granularity: Optional[str] = Field(
|
876
884
|
None,
|
877
|
-
description=
|
878
|
-
examples=[
|
879
|
-
title=
|
885
|
+
description='Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.',
|
886
|
+
examples=['PT1S'],
|
887
|
+
title='Cursor Granularity',
|
880
888
|
)
|
881
889
|
end_datetime: Optional[Union[str, MinMaxDatetime]] = Field(
|
882
890
|
None,
|
883
|
-
description=
|
884
|
-
examples=[
|
885
|
-
title=
|
891
|
+
description='The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.',
|
892
|
+
examples=['2021-01-1T00:00:00Z', '{{ now_utc() }}', '{{ day_delta(-1) }}'],
|
893
|
+
title='End Datetime',
|
886
894
|
)
|
887
895
|
end_time_option: Optional[RequestOption] = Field(
|
888
896
|
None,
|
889
|
-
description=
|
890
|
-
title=
|
897
|
+
description='Optionally configures how the end datetime will be sent in requests to the source API.',
|
898
|
+
title='Inject End Time Into Outgoing HTTP Request',
|
891
899
|
)
|
892
900
|
is_data_feed: Optional[bool] = Field(
|
893
901
|
None,
|
894
|
-
description=
|
895
|
-
title=
|
902
|
+
description='A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.',
|
903
|
+
title='Whether the target API is formatted as a data feed',
|
896
904
|
)
|
897
905
|
lookback_window: Optional[str] = Field(
|
898
906
|
None,
|
899
|
-
description=
|
900
|
-
examples=[
|
901
|
-
title=
|
907
|
+
description='Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.',
|
908
|
+
examples=['P1D', "P{{ config['lookback_days'] }}D"],
|
909
|
+
title='Lookback Window',
|
902
910
|
)
|
903
911
|
partition_field_end: Optional[str] = Field(
|
904
912
|
None,
|
905
|
-
description=
|
906
|
-
examples=[
|
907
|
-
title=
|
913
|
+
description='Name of the partition start time field.',
|
914
|
+
examples=['ending_time'],
|
915
|
+
title='Partition Field End',
|
908
916
|
)
|
909
917
|
partition_field_start: Optional[str] = Field(
|
910
918
|
None,
|
911
|
-
description=
|
912
|
-
examples=[
|
913
|
-
title=
|
919
|
+
description='Name of the partition end time field.',
|
920
|
+
examples=['starting_time'],
|
921
|
+
title='Partition Field Start',
|
914
922
|
)
|
915
923
|
start_time_option: Optional[RequestOption] = Field(
|
916
924
|
None,
|
917
|
-
description=
|
918
|
-
title=
|
925
|
+
description='Optionally configures how the start datetime will be sent in requests to the source API.',
|
926
|
+
title='Inject Start Time Into Outgoing HTTP Request',
|
919
927
|
)
|
920
928
|
step: Optional[str] = Field(
|
921
929
|
None,
|
922
|
-
description=
|
923
|
-
examples=[
|
924
|
-
title=
|
930
|
+
description='The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.',
|
931
|
+
examples=['P1W', "{{ config['step_increment'] }}"],
|
932
|
+
title='Step',
|
925
933
|
)
|
926
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
934
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
927
935
|
|
928
936
|
|
929
937
|
class DefaultErrorHandler(BaseModel):
|
930
|
-
type: Literal[
|
938
|
+
type: Literal['DefaultErrorHandler']
|
931
939
|
backoff_strategies: Optional[
|
932
940
|
List[
|
933
941
|
Union[
|
@@ -940,143 +948,145 @@ class DefaultErrorHandler(BaseModel):
|
|
940
948
|
]
|
941
949
|
] = Field(
|
942
950
|
None,
|
943
|
-
description=
|
944
|
-
title=
|
951
|
+
description='List of backoff strategies to use to determine how long to wait before retrying a retryable request.',
|
952
|
+
title='Backoff Strategies',
|
945
953
|
)
|
946
954
|
max_retries: Optional[int] = Field(
|
947
955
|
5,
|
948
|
-
description=
|
956
|
+
description='The maximum number of time to retry a retryable request before giving up and failing.',
|
949
957
|
examples=[5, 0, 10],
|
950
|
-
title=
|
958
|
+
title='Max Retry Count',
|
951
959
|
)
|
952
960
|
response_filters: Optional[List[HttpResponseFilter]] = Field(
|
953
961
|
None,
|
954
962
|
description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.",
|
955
|
-
title=
|
963
|
+
title='Response Filters',
|
956
964
|
)
|
957
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
965
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
958
966
|
|
959
967
|
|
960
968
|
class DefaultPaginator(BaseModel):
|
961
|
-
type: Literal[
|
962
|
-
pagination_strategy: Union[
|
969
|
+
type: Literal['DefaultPaginator']
|
970
|
+
pagination_strategy: Union[
|
971
|
+
CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement
|
972
|
+
] = Field(
|
963
973
|
...,
|
964
|
-
description=
|
965
|
-
title=
|
974
|
+
description='Strategy defining how records are paginated.',
|
975
|
+
title='Pagination Strategy',
|
966
976
|
)
|
967
977
|
decoder: Optional[JsonDecoder] = Field(
|
968
978
|
None,
|
969
|
-
description=
|
970
|
-
title=
|
979
|
+
description='Component decoding the response so records can be extracted.',
|
980
|
+
title='Decoder',
|
971
981
|
)
|
972
982
|
page_size_option: Optional[RequestOption] = None
|
973
983
|
page_token_option: Optional[Union[RequestOption, RequestPath]] = None
|
974
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
984
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
975
985
|
|
976
986
|
|
977
987
|
class DpathExtractor(BaseModel):
|
978
|
-
type: Literal[
|
988
|
+
type: Literal['DpathExtractor']
|
979
989
|
field_path: List[str] = Field(
|
980
990
|
...,
|
981
991
|
description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
|
982
992
|
examples=[
|
983
|
-
[
|
984
|
-
[
|
985
|
-
[
|
986
|
-
[
|
993
|
+
['data'],
|
994
|
+
['data', 'records'],
|
995
|
+
['data', '{{ parameters.name }}'],
|
996
|
+
['data', '*', 'record'],
|
987
997
|
],
|
988
|
-
title=
|
998
|
+
title='Field Path',
|
989
999
|
)
|
990
1000
|
decoder: Optional[JsonDecoder] = Field(
|
991
1001
|
None,
|
992
|
-
description=
|
993
|
-
title=
|
1002
|
+
description='Component decoding the response so records can be extracted.',
|
1003
|
+
title='Decoder',
|
994
1004
|
)
|
995
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1005
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
996
1006
|
|
997
1007
|
|
998
1008
|
class SessionTokenRequestApiKeyAuthenticator(BaseModel):
|
999
|
-
type: Literal[
|
1009
|
+
type: Literal['ApiKey']
|
1000
1010
|
inject_into: RequestOption = Field(
|
1001
1011
|
...,
|
1002
|
-
description=
|
1012
|
+
description='Configure how the API Key will be sent in requests to the source API.',
|
1003
1013
|
examples=[
|
1004
|
-
{
|
1005
|
-
{
|
1014
|
+
{'inject_into': 'header', 'field_name': 'Authorization'},
|
1015
|
+
{'inject_into': 'request_parameter', 'field_name': 'authKey'},
|
1006
1016
|
],
|
1007
|
-
title=
|
1017
|
+
title='Inject API Key Into Outgoing HTTP Request',
|
1008
1018
|
)
|
1009
1019
|
|
1010
1020
|
|
1011
1021
|
class ListPartitionRouter(BaseModel):
|
1012
|
-
type: Literal[
|
1022
|
+
type: Literal['ListPartitionRouter']
|
1013
1023
|
cursor_field: str = Field(
|
1014
1024
|
...,
|
1015
1025
|
description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.',
|
1016
|
-
examples=[
|
1017
|
-
title=
|
1026
|
+
examples=['section', "{{ config['section_key'] }}"],
|
1027
|
+
title='Current Partition Value Identifier',
|
1018
1028
|
)
|
1019
1029
|
values: Union[str, List[str]] = Field(
|
1020
1030
|
...,
|
1021
|
-
description=
|
1022
|
-
examples=[[
|
1023
|
-
title=
|
1031
|
+
description='The list of attributes being iterated over and used as input for the requests made to the source API.',
|
1032
|
+
examples=[['section_a', 'section_b', 'section_c'], "{{ config['sections'] }}"],
|
1033
|
+
title='Partition Values',
|
1024
1034
|
)
|
1025
1035
|
request_option: Optional[RequestOption] = Field(
|
1026
1036
|
None,
|
1027
|
-
description=
|
1028
|
-
title=
|
1037
|
+
description='A request option describing where the list value should be injected into and under what field name if applicable.',
|
1038
|
+
title='Inject Partition Value Into Outgoing HTTP Request',
|
1029
1039
|
)
|
1030
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1040
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1031
1041
|
|
1032
1042
|
|
1033
1043
|
class RecordSelector(BaseModel):
|
1034
|
-
type: Literal[
|
1044
|
+
type: Literal['RecordSelector']
|
1035
1045
|
extractor: Union[CustomRecordExtractor, DpathExtractor]
|
1036
|
-
record_filter: Optional[Union[
|
1046
|
+
record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field(
|
1037
1047
|
None,
|
1038
|
-
description=
|
1039
|
-
title=
|
1048
|
+
description='Responsible for filtering records to be emitted by the Source.',
|
1049
|
+
title='Record Filter',
|
1040
1050
|
)
|
1041
1051
|
schema_normalization: Optional[SchemaNormalization] = SchemaNormalization.None_
|
1042
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1052
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1043
1053
|
|
1044
1054
|
|
1045
1055
|
class Spec(BaseModel):
|
1046
|
-
type: Literal[
|
1056
|
+
type: Literal['Spec']
|
1047
1057
|
connection_specification: Dict[str, Any] = Field(
|
1048
1058
|
...,
|
1049
|
-
description=
|
1050
|
-
title=
|
1059
|
+
description='A connection specification describing how a the connector can be configured.',
|
1060
|
+
title='Connection Specification',
|
1051
1061
|
)
|
1052
1062
|
documentation_url: Optional[str] = Field(
|
1053
1063
|
None,
|
1054
1064
|
description="URL of the connector's documentation page.",
|
1055
|
-
examples=[
|
1056
|
-
title=
|
1065
|
+
examples=['https://docs.airbyte.com/integrations/sources/dremio'],
|
1066
|
+
title='Documentation URL',
|
1057
1067
|
)
|
1058
1068
|
advanced_auth: Optional[AuthFlow] = Field(
|
1059
1069
|
None,
|
1060
|
-
description=
|
1061
|
-
title=
|
1070
|
+
description='Advanced specification for configuring the authentication flow.',
|
1071
|
+
title='Advanced Auth',
|
1062
1072
|
)
|
1063
1073
|
|
1064
1074
|
|
1065
1075
|
class CompositeErrorHandler(BaseModel):
|
1066
|
-
type: Literal[
|
1076
|
+
type: Literal['CompositeErrorHandler']
|
1067
1077
|
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field(
|
1068
1078
|
...,
|
1069
|
-
description=
|
1070
|
-
title=
|
1079
|
+
description='List of error handlers to iterate on to determine how to handle a failed response.',
|
1080
|
+
title='Error Handlers',
|
1071
1081
|
)
|
1072
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1082
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1073
1083
|
|
1074
1084
|
|
1075
1085
|
class DeclarativeSource(BaseModel):
|
1076
1086
|
class Config:
|
1077
1087
|
extra = Extra.forbid
|
1078
1088
|
|
1079
|
-
type: Literal[
|
1089
|
+
type: Literal['DeclarativeSource']
|
1080
1090
|
check: CheckStream
|
1081
1091
|
streams: List[DeclarativeStream]
|
1082
1092
|
version: str
|
@@ -1085,7 +1095,7 @@ class DeclarativeSource(BaseModel):
|
|
1085
1095
|
spec: Optional[Spec] = None
|
1086
1096
|
metadata: Optional[Dict[str, Any]] = Field(
|
1087
1097
|
None,
|
1088
|
-
description=
|
1098
|
+
description='For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.',
|
1089
1099
|
)
|
1090
1100
|
|
1091
1101
|
|
@@ -1093,12 +1103,12 @@ class SelectiveAuthenticator(BaseModel):
|
|
1093
1103
|
class Config:
|
1094
1104
|
extra = Extra.allow
|
1095
1105
|
|
1096
|
-
type: Literal[
|
1106
|
+
type: Literal['SelectiveAuthenticator']
|
1097
1107
|
authenticator_selection_path: List[str] = Field(
|
1098
1108
|
...,
|
1099
|
-
description=
|
1100
|
-
examples=[[
|
1101
|
-
title=
|
1109
|
+
description='Path of the field in config with selected authenticator name',
|
1110
|
+
examples=[['auth'], ['auth', 'type']],
|
1111
|
+
title='Authenticator Selection Path',
|
1102
1112
|
)
|
1103
1113
|
authenticators: Dict[
|
1104
1114
|
str,
|
@@ -1114,109 +1124,119 @@ class SelectiveAuthenticator(BaseModel):
|
|
1114
1124
|
],
|
1115
1125
|
] = Field(
|
1116
1126
|
...,
|
1117
|
-
description=
|
1127
|
+
description='Authenticators to select from.',
|
1118
1128
|
examples=[
|
1119
1129
|
{
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1130
|
+
'authenticators': {
|
1131
|
+
'token': '#/definitions/ApiKeyAuthenticator',
|
1132
|
+
'oauth': '#/definitions/OAuthAuthenticator',
|
1123
1133
|
}
|
1124
1134
|
}
|
1125
1135
|
],
|
1126
|
-
title=
|
1136
|
+
title='Authenticators',
|
1127
1137
|
)
|
1128
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1138
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1129
1139
|
|
1130
1140
|
|
1131
1141
|
class DeclarativeStream(BaseModel):
|
1132
1142
|
class Config:
|
1133
1143
|
extra = Extra.allow
|
1134
1144
|
|
1135
|
-
type: Literal[
|
1145
|
+
type: Literal['DeclarativeStream']
|
1136
1146
|
retriever: Union[CustomRetriever, SimpleRetriever] = Field(
|
1137
1147
|
...,
|
1138
|
-
description=
|
1139
|
-
title=
|
1148
|
+
description='Component used to coordinate how records are extracted across stream slices and request pages.',
|
1149
|
+
title='Retriever',
|
1140
1150
|
)
|
1141
|
-
incremental_sync: Optional[
|
1151
|
+
incremental_sync: Optional[
|
1152
|
+
Union[CustomIncrementalSync, DatetimeBasedCursor]
|
1153
|
+
] = Field(
|
1142
1154
|
None,
|
1143
|
-
description=
|
1144
|
-
title=
|
1155
|
+
description='Component used to fetch data incrementally based on a time field in the data.',
|
1156
|
+
title='Incremental Sync',
|
1157
|
+
)
|
1158
|
+
name: Optional[str] = Field(
|
1159
|
+
'', description='The stream name.', example=['Users'], title='Name'
|
1160
|
+
)
|
1161
|
+
primary_key: Optional[PrimaryKey] = Field(
|
1162
|
+
'', description='The primary key of the stream.', title='Primary Key'
|
1145
1163
|
)
|
1146
|
-
name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name")
|
1147
|
-
primary_key: Optional[PrimaryKey] = Field("", description="The primary key of the stream.", title="Primary Key")
|
1148
1164
|
schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader]] = Field(
|
1149
1165
|
None,
|
1150
|
-
description=
|
1151
|
-
title=
|
1166
|
+
description='Component used to retrieve the schema for the current stream.',
|
1167
|
+
title='Schema Loader',
|
1152
1168
|
)
|
1153
|
-
transformations: Optional[
|
1169
|
+
transformations: Optional[
|
1170
|
+
List[Union[AddFields, CustomTransformation, RemoveFields]]
|
1171
|
+
] = Field(
|
1154
1172
|
None,
|
1155
|
-
description=
|
1156
|
-
title=
|
1173
|
+
description='A list of transformations to be applied to each output record.',
|
1174
|
+
title='Transformations',
|
1157
1175
|
)
|
1158
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1176
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1159
1177
|
|
1160
1178
|
|
1161
1179
|
class SessionTokenAuthenticator(BaseModel):
|
1162
|
-
type: Literal[
|
1180
|
+
type: Literal['SessionTokenAuthenticator']
|
1163
1181
|
login_requester: HttpRequester = Field(
|
1164
1182
|
...,
|
1165
|
-
description=
|
1183
|
+
description='Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.',
|
1166
1184
|
examples=[
|
1167
1185
|
{
|
1168
|
-
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1174
|
-
|
1186
|
+
'type': 'HttpRequester',
|
1187
|
+
'url_base': 'https://my_api.com',
|
1188
|
+
'path': '/login',
|
1189
|
+
'authenticator': {
|
1190
|
+
'type': 'BasicHttpAuthenticator',
|
1191
|
+
'username': '{{ config.username }}',
|
1192
|
+
'password': '{{ config.password }}',
|
1175
1193
|
},
|
1176
1194
|
}
|
1177
1195
|
],
|
1178
|
-
title=
|
1196
|
+
title='Login Requester',
|
1179
1197
|
)
|
1180
1198
|
session_token_path: List[str] = Field(
|
1181
1199
|
...,
|
1182
|
-
description=
|
1183
|
-
examples=[[
|
1184
|
-
title=
|
1200
|
+
description='The path in the response body returned from the login requester to the session token.',
|
1201
|
+
examples=[['access_token'], ['result', 'token']],
|
1202
|
+
title='Session Token Path',
|
1185
1203
|
)
|
1186
1204
|
expiration_duration: Optional[str] = Field(
|
1187
1205
|
None,
|
1188
|
-
description=
|
1189
|
-
examples=[
|
1190
|
-
title=
|
1206
|
+
description='The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.',
|
1207
|
+
examples=['PT1H', 'P1D'],
|
1208
|
+
title='Expiration Duration',
|
1191
1209
|
)
|
1192
|
-
request_authentication: Union[
|
1210
|
+
request_authentication: Union[
|
1211
|
+
SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator
|
1212
|
+
] = Field(
|
1193
1213
|
...,
|
1194
|
-
description=
|
1195
|
-
title=
|
1214
|
+
description='Authentication method to use for requests sent to the API, specifying how to inject the session token.',
|
1215
|
+
title='Data Request Authentication',
|
1196
1216
|
)
|
1197
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1217
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1198
1218
|
|
1199
1219
|
|
1200
1220
|
class HttpRequester(BaseModel):
|
1201
|
-
type: Literal[
|
1221
|
+
type: Literal['HttpRequester']
|
1202
1222
|
url_base: str = Field(
|
1203
1223
|
...,
|
1204
|
-
description=
|
1224
|
+
description='Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.',
|
1205
1225
|
examples=[
|
1206
|
-
|
1226
|
+
'https://connect.squareup.com/v2',
|
1207
1227
|
"{{ config['base_url'] or 'https://app.posthog.com'}}/api/",
|
1208
1228
|
],
|
1209
|
-
title=
|
1229
|
+
title='API Base URL',
|
1210
1230
|
)
|
1211
1231
|
path: str = Field(
|
1212
1232
|
...,
|
1213
|
-
description=
|
1233
|
+
description='Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.',
|
1214
1234
|
examples=[
|
1215
|
-
|
1235
|
+
'/products',
|
1216
1236
|
"/quotes/{{ stream_partition['id'] }}/quote_line_groups",
|
1217
1237
|
"/trades/{{ config['symbol_id'] }}/history",
|
1218
1238
|
],
|
1219
|
-
title=
|
1239
|
+
title='URL Path',
|
1220
1240
|
)
|
1221
1241
|
authenticator: Optional[
|
1222
1242
|
Union[
|
@@ -1232,97 +1252,101 @@ class HttpRequester(BaseModel):
|
|
1232
1252
|
]
|
1233
1253
|
] = Field(
|
1234
1254
|
None,
|
1235
|
-
description=
|
1236
|
-
title=
|
1255
|
+
description='Authentication method to use for requests sent to the API.',
|
1256
|
+
title='Authenticator',
|
1237
1257
|
)
|
1238
|
-
error_handler: Optional[
|
1258
|
+
error_handler: Optional[
|
1259
|
+
Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]
|
1260
|
+
] = Field(
|
1239
1261
|
None,
|
1240
|
-
description=
|
1241
|
-
title=
|
1262
|
+
description='Error handler component that defines how to handle errors.',
|
1263
|
+
title='Error Handler',
|
1242
1264
|
)
|
1243
1265
|
http_method: Optional[HttpMethod] = Field(
|
1244
1266
|
HttpMethod.GET,
|
1245
|
-
description=
|
1246
|
-
examples=[
|
1247
|
-
title=
|
1267
|
+
description='The HTTP method used to fetch data from the source (can be GET or POST).',
|
1268
|
+
examples=['GET', 'POST'],
|
1269
|
+
title='HTTP Method',
|
1248
1270
|
)
|
1249
1271
|
request_body_data: Optional[Union[str, Dict[str, str]]] = Field(
|
1250
1272
|
None,
|
1251
|
-
description=
|
1273
|
+
description='Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.',
|
1252
1274
|
examples=[
|
1253
1275
|
'[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n'
|
1254
1276
|
],
|
1255
|
-
title=
|
1277
|
+
title='Request Body Payload (Non-JSON)',
|
1256
1278
|
)
|
1257
1279
|
request_body_json: Optional[Union[str, Dict[str, Any]]] = Field(
|
1258
1280
|
None,
|
1259
|
-
description=
|
1281
|
+
description='Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.',
|
1260
1282
|
examples=[
|
1261
|
-
{
|
1262
|
-
{
|
1263
|
-
{
|
1283
|
+
{'sort_order': 'ASC', 'sort_field': 'CREATED_AT'},
|
1284
|
+
{'key': "{{ config['value'] }}"},
|
1285
|
+
{'sort': {'field': 'updated_at', 'order': 'ascending'}},
|
1264
1286
|
],
|
1265
|
-
title=
|
1287
|
+
title='Request Body JSON Payload',
|
1266
1288
|
)
|
1267
1289
|
request_headers: Optional[Union[str, Dict[str, str]]] = Field(
|
1268
1290
|
None,
|
1269
|
-
description=
|
1270
|
-
examples=[{
|
1271
|
-
title=
|
1291
|
+
description='Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.',
|
1292
|
+
examples=[{'Output-Format': 'JSON'}, {'Version': "{{ config['version'] }}"}],
|
1293
|
+
title='Request Headers',
|
1272
1294
|
)
|
1273
1295
|
request_parameters: Optional[Union[str, Dict[str, str]]] = Field(
|
1274
1296
|
None,
|
1275
|
-
description=
|
1297
|
+
description='Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.',
|
1276
1298
|
examples=[
|
1277
|
-
{
|
1299
|
+
{'unit': 'day'},
|
1278
1300
|
{
|
1279
|
-
|
1301
|
+
'query': 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
|
1280
1302
|
},
|
1281
|
-
{
|
1282
|
-
{
|
1303
|
+
{'searchIn': "{{ ','.join(config.get('search_in', [])) }}"},
|
1304
|
+
{'sort_by[asc]': 'updated_at'},
|
1283
1305
|
],
|
1284
|
-
title=
|
1306
|
+
title='Query Parameters',
|
1285
1307
|
)
|
1286
1308
|
use_cache: Optional[bool] = Field(
|
1287
1309
|
False,
|
1288
|
-
description=
|
1289
|
-
title=
|
1310
|
+
description='Enables stream requests caching. This field is automatically set by the CDK.',
|
1311
|
+
title='Use Cache',
|
1290
1312
|
)
|
1291
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1313
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1292
1314
|
|
1293
1315
|
|
1294
1316
|
class ParentStreamConfig(BaseModel):
|
1295
|
-
type: Literal[
|
1317
|
+
type: Literal['ParentStreamConfig']
|
1296
1318
|
parent_key: str = Field(
|
1297
1319
|
...,
|
1298
|
-
description=
|
1299
|
-
examples=[
|
1300
|
-
title=
|
1320
|
+
description='The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.',
|
1321
|
+
examples=['id', "{{ config['parent_record_id'] }}"],
|
1322
|
+
title='Parent Key',
|
1323
|
+
)
|
1324
|
+
stream: DeclarativeStream = Field(
|
1325
|
+
..., description='Reference to the parent stream.', title='Parent Stream'
|
1301
1326
|
)
|
1302
|
-
stream: DeclarativeStream = Field(..., description="Reference to the parent stream.", title="Parent Stream")
|
1303
1327
|
partition_field: str = Field(
|
1304
1328
|
...,
|
1305
|
-
description=
|
1306
|
-
examples=[
|
1307
|
-
title=
|
1329
|
+
description='While iterating over parent records during a sync, the parent_key value can be referenced by using this field.',
|
1330
|
+
examples=['parent_id', "{{ config['parent_partition_field'] }}"],
|
1331
|
+
title='Current Parent Key Value Identifier',
|
1308
1332
|
)
|
1309
1333
|
request_option: Optional[RequestOption] = Field(
|
1310
1334
|
None,
|
1311
|
-
description=
|
1312
|
-
title=
|
1335
|
+
description='A request option describing where the parent key value should be injected into and under what field name if applicable.',
|
1336
|
+
title='Request Option',
|
1313
1337
|
)
|
1314
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1338
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1315
1339
|
|
1316
1340
|
|
1317
1341
|
class SimpleRetriever(BaseModel):
|
1318
|
-
type: Literal[
|
1342
|
+
type: Literal['SimpleRetriever']
|
1319
1343
|
record_selector: RecordSelector = Field(
|
1320
1344
|
...,
|
1321
|
-
description=
|
1345
|
+
description='Component that describes how to extract records from a HTTP response.',
|
1322
1346
|
)
|
1323
1347
|
requester: Union[CustomRequester, HttpRequester] = Field(
|
1324
1348
|
...,
|
1325
|
-
description=
|
1349
|
+
description='Requester component that describes how to prepare HTTP requests to send to the source API.',
|
1326
1350
|
)
|
1327
1351
|
paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
|
1328
1352
|
None,
|
@@ -1337,24 +1361,28 @@ class SimpleRetriever(BaseModel):
|
|
1337
1361
|
CustomPartitionRouter,
|
1338
1362
|
ListPartitionRouter,
|
1339
1363
|
SubstreamPartitionRouter,
|
1340
|
-
List[
|
1364
|
+
List[
|
1365
|
+
Union[
|
1366
|
+
CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
|
1367
|
+
]
|
1368
|
+
],
|
1341
1369
|
]
|
1342
1370
|
] = Field(
|
1343
1371
|
[],
|
1344
|
-
description=
|
1345
|
-
title=
|
1372
|
+
description='PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.',
|
1373
|
+
title='Partition Router',
|
1346
1374
|
)
|
1347
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1375
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1348
1376
|
|
1349
1377
|
|
1350
1378
|
class SubstreamPartitionRouter(BaseModel):
|
1351
|
-
type: Literal[
|
1379
|
+
type: Literal['SubstreamPartitionRouter']
|
1352
1380
|
parent_stream_configs: List[ParentStreamConfig] = Field(
|
1353
1381
|
...,
|
1354
|
-
description=
|
1355
|
-
title=
|
1382
|
+
description='Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.',
|
1383
|
+
title='Parent Stream Configs',
|
1356
1384
|
)
|
1357
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1385
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
|
1358
1386
|
|
1359
1387
|
|
1360
1388
|
CompositeErrorHandler.update_forward_refs()
|