airbyte-cdk 0.51.16__py3-none-any.whl → 0.51.17__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +494 -522
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +1 -1
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +2 -37
- airbyte_cdk/sources/file_based/file_based_source.py +1 -1
- airbyte_cdk/sources/file_based/file_types/__init__.py +11 -6
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +1 -1
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +1 -1
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +2 -2
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +5 -5
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +7 -5
- {airbyte_cdk-0.51.16.dist-info → airbyte_cdk-0.51.17.dist-info}/METADATA +1 -1
- {airbyte_cdk-0.51.16.dist-info → airbyte_cdk-0.51.17.dist-info}/RECORD +27 -27
- unit_tests/sources/file_based/file_types/test_avro_parser.py +6 -6
- unit_tests/sources/file_based/scenarios/avro_scenarios.py +5 -6
- unit_tests/sources/file_based/scenarios/check_scenarios.py +8 -8
- unit_tests/sources/file_based/scenarios/csv_scenarios.py +19 -42
- unit_tests/sources/file_based/scenarios/incremental_scenarios.py +15 -15
- unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +13 -12
- unit_tests/sources/file_based/scenarios/parquet_scenarios.py +5 -9
- unit_tests/sources/file_based/scenarios/scenario_builder.py +1 -1
- unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +16 -16
- unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +9 -9
- unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +2 -1
- unit_tests/sources/file_based/stream/test_default_file_based_stream.py +6 -3
- {airbyte_cdk-0.51.16.dist-info → airbyte_cdk-0.51.17.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-0.51.16.dist-info → airbyte_cdk-0.51.17.dist-info}/WHEEL +0 -0
- {airbyte_cdk-0.51.16.dist-info → airbyte_cdk-0.51.17.dist-info}/top_level.txt +0 -0
@@ -11,12 +11,12 @@ from typing_extensions import Literal
|
|
11
11
|
|
12
12
|
|
13
13
|
class AddedFieldDefinition(BaseModel):
|
14
|
-
type: Literal[
|
14
|
+
type: Literal["AddedFieldDefinition"]
|
15
15
|
path: List[str] = Field(
|
16
16
|
...,
|
17
|
-
description=
|
18
|
-
examples=[[
|
19
|
-
title=
|
17
|
+
description="List of strings defining the path where to add the value on the record.",
|
18
|
+
examples=[["segment_id"], ["metadata", "segment_id"]],
|
19
|
+
title="Path",
|
20
20
|
)
|
21
21
|
value: str = Field(
|
22
22
|
...,
|
@@ -26,607 +26,601 @@ class AddedFieldDefinition(BaseModel):
|
|
26
26
|
"{{ record['MetaData']['LastUpdatedTime'] }}",
|
27
27
|
"{{ stream_partition['segment_id'] }}",
|
28
28
|
],
|
29
|
-
title=
|
29
|
+
title="Value",
|
30
30
|
)
|
31
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
31
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
32
32
|
|
33
33
|
|
34
34
|
class AddFields(BaseModel):
|
35
|
-
type: Literal[
|
35
|
+
type: Literal["AddFields"]
|
36
36
|
fields: List[AddedFieldDefinition] = Field(
|
37
37
|
...,
|
38
|
-
description=
|
39
|
-
title=
|
38
|
+
description="List of transformations (path and corresponding value) that will be added to the record.",
|
39
|
+
title="Fields",
|
40
40
|
)
|
41
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
41
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
42
42
|
|
43
43
|
|
44
44
|
class AuthFlowType(Enum):
|
45
|
-
oauth2_0 =
|
46
|
-
oauth1_0 =
|
45
|
+
oauth2_0 = "oauth2.0"
|
46
|
+
oauth1_0 = "oauth1.0"
|
47
47
|
|
48
48
|
|
49
49
|
class BasicHttpAuthenticator(BaseModel):
|
50
|
-
type: Literal[
|
50
|
+
type: Literal["BasicHttpAuthenticator"]
|
51
51
|
username: str = Field(
|
52
52
|
...,
|
53
|
-
description=
|
53
|
+
description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.",
|
54
54
|
examples=["{{ config['username'] }}", "{{ config['api_key'] }}"],
|
55
|
-
title=
|
55
|
+
title="Username",
|
56
56
|
)
|
57
57
|
password: Optional[str] = Field(
|
58
|
-
|
59
|
-
description=
|
60
|
-
examples=["{{ config['password'] }}",
|
61
|
-
title=
|
58
|
+
"",
|
59
|
+
description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.",
|
60
|
+
examples=["{{ config['password'] }}", ""],
|
61
|
+
title="Password",
|
62
62
|
)
|
63
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
63
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
64
64
|
|
65
65
|
|
66
66
|
class BearerAuthenticator(BaseModel):
|
67
|
-
type: Literal[
|
67
|
+
type: Literal["BearerAuthenticator"]
|
68
68
|
api_token: str = Field(
|
69
69
|
...,
|
70
|
-
description=
|
70
|
+
description="Token to inject as request header for authenticating with the API.",
|
71
71
|
examples=["{{ config['api_key'] }}", "{{ config['token'] }}"],
|
72
|
-
title=
|
72
|
+
title="Bearer Token",
|
73
73
|
)
|
74
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
74
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
75
75
|
|
76
76
|
|
77
77
|
class CheckStream(BaseModel):
|
78
|
-
type: Literal[
|
78
|
+
type: Literal["CheckStream"]
|
79
79
|
stream_names: List[str] = Field(
|
80
80
|
...,
|
81
|
-
description=
|
82
|
-
examples=[[
|
83
|
-
title=
|
81
|
+
description="Names of the streams to try reading from when running a check operation.",
|
82
|
+
examples=[["users"], ["users", "contacts"]],
|
83
|
+
title="Stream Names",
|
84
84
|
)
|
85
85
|
|
86
86
|
|
87
87
|
class ConstantBackoffStrategy(BaseModel):
|
88
|
-
type: Literal[
|
88
|
+
type: Literal["ConstantBackoffStrategy"]
|
89
89
|
backoff_time_in_seconds: Union[float, str] = Field(
|
90
90
|
...,
|
91
|
-
description=
|
91
|
+
description="Backoff time in seconds.",
|
92
92
|
examples=[30, 30.5, "{{ config['backoff_time'] }}"],
|
93
|
-
title=
|
93
|
+
title="Backoff Time",
|
94
94
|
)
|
95
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
95
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
96
96
|
|
97
97
|
|
98
98
|
class CustomAuthenticator(BaseModel):
|
99
99
|
class Config:
|
100
100
|
extra = Extra.allow
|
101
101
|
|
102
|
-
type: Literal[
|
102
|
+
type: Literal["CustomAuthenticator"]
|
103
103
|
class_name: str = Field(
|
104
104
|
...,
|
105
|
-
description=
|
106
|
-
examples=[
|
107
|
-
title=
|
105
|
+
description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_<name>.<package>.<class_name>`.",
|
106
|
+
examples=["source_railz.components.ShortLivedTokenAuthenticator"],
|
107
|
+
title="Class Name",
|
108
108
|
)
|
109
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
109
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
110
110
|
|
111
111
|
|
112
112
|
class CustomBackoffStrategy(BaseModel):
|
113
113
|
class Config:
|
114
114
|
extra = Extra.allow
|
115
115
|
|
116
|
-
type: Literal[
|
116
|
+
type: Literal["CustomBackoffStrategy"]
|
117
117
|
class_name: str = Field(
|
118
118
|
...,
|
119
|
-
description=
|
120
|
-
examples=[
|
121
|
-
title=
|
119
|
+
description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_<name>.<package>.<class_name>`.",
|
120
|
+
examples=["source_railz.components.MyCustomBackoffStrategy"],
|
121
|
+
title="Class Name",
|
122
122
|
)
|
123
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
123
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
124
124
|
|
125
125
|
|
126
126
|
class CustomErrorHandler(BaseModel):
|
127
127
|
class Config:
|
128
128
|
extra = Extra.allow
|
129
129
|
|
130
|
-
type: Literal[
|
130
|
+
type: Literal["CustomErrorHandler"]
|
131
131
|
class_name: str = Field(
|
132
132
|
...,
|
133
|
-
description=
|
134
|
-
examples=[
|
135
|
-
title=
|
133
|
+
description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_<name>.<package>.<class_name>`.",
|
134
|
+
examples=["source_railz.components.MyCustomErrorHandler"],
|
135
|
+
title="Class Name",
|
136
136
|
)
|
137
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
137
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
138
138
|
|
139
139
|
|
140
140
|
class CustomIncrementalSync(BaseModel):
|
141
141
|
class Config:
|
142
142
|
extra = Extra.allow
|
143
143
|
|
144
|
-
type: Literal[
|
144
|
+
type: Literal["CustomIncrementalSync"]
|
145
145
|
class_name: str = Field(
|
146
146
|
...,
|
147
|
-
description=
|
148
|
-
examples=[
|
149
|
-
title=
|
147
|
+
description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_<name>.<package>.<class_name>`.",
|
148
|
+
examples=["source_railz.components.MyCustomIncrementalSync"],
|
149
|
+
title="Class Name",
|
150
150
|
)
|
151
151
|
cursor_field: str = Field(
|
152
152
|
...,
|
153
|
-
description=
|
153
|
+
description="The location of the value on a record that will be used as a bookmark during sync.",
|
154
154
|
)
|
155
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
155
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
156
156
|
|
157
157
|
|
158
158
|
class CustomPaginationStrategy(BaseModel):
|
159
159
|
class Config:
|
160
160
|
extra = Extra.allow
|
161
161
|
|
162
|
-
type: Literal[
|
162
|
+
type: Literal["CustomPaginationStrategy"]
|
163
163
|
class_name: str = Field(
|
164
164
|
...,
|
165
|
-
description=
|
166
|
-
examples=[
|
167
|
-
title=
|
165
|
+
description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_<name>.<package>.<class_name>`.",
|
166
|
+
examples=["source_railz.components.MyCustomPaginationStrategy"],
|
167
|
+
title="Class Name",
|
168
168
|
)
|
169
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
169
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
170
170
|
|
171
171
|
|
172
172
|
class CustomRecordExtractor(BaseModel):
|
173
173
|
class Config:
|
174
174
|
extra = Extra.allow
|
175
175
|
|
176
|
-
type: Literal[
|
176
|
+
type: Literal["CustomRecordExtractor"]
|
177
177
|
class_name: str = Field(
|
178
178
|
...,
|
179
|
-
description=
|
180
|
-
examples=[
|
181
|
-
title=
|
179
|
+
description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_<name>.<package>.<class_name>`.",
|
180
|
+
examples=["source_railz.components.MyCustomRecordExtractor"],
|
181
|
+
title="Class Name",
|
182
182
|
)
|
183
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
183
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
184
184
|
|
185
185
|
|
186
186
|
class CustomRequester(BaseModel):
|
187
187
|
class Config:
|
188
188
|
extra = Extra.allow
|
189
189
|
|
190
|
-
type: Literal[
|
190
|
+
type: Literal["CustomRequester"]
|
191
191
|
class_name: str = Field(
|
192
192
|
...,
|
193
|
-
description=
|
194
|
-
examples=[
|
195
|
-
title=
|
193
|
+
description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_<name>.<package>.<class_name>`.",
|
194
|
+
examples=["source_railz.components.MyCustomRecordExtractor"],
|
195
|
+
title="Class Name",
|
196
196
|
)
|
197
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
197
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
198
198
|
|
199
199
|
|
200
200
|
class CustomRetriever(BaseModel):
|
201
201
|
class Config:
|
202
202
|
extra = Extra.allow
|
203
203
|
|
204
|
-
type: Literal[
|
204
|
+
type: Literal["CustomRetriever"]
|
205
205
|
class_name: str = Field(
|
206
206
|
...,
|
207
|
-
description=
|
208
|
-
examples=[
|
209
|
-
title=
|
207
|
+
description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_<name>.<package>.<class_name>`.",
|
208
|
+
examples=["source_railz.components.MyCustomRetriever"],
|
209
|
+
title="Class Name",
|
210
210
|
)
|
211
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
211
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
212
212
|
|
213
213
|
|
214
214
|
class CustomPartitionRouter(BaseModel):
|
215
215
|
class Config:
|
216
216
|
extra = Extra.allow
|
217
217
|
|
218
|
-
type: Literal[
|
218
|
+
type: Literal["CustomPartitionRouter"]
|
219
219
|
class_name: str = Field(
|
220
220
|
...,
|
221
|
-
description=
|
222
|
-
examples=[
|
223
|
-
title=
|
221
|
+
description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_<name>.<package>.<class_name>`.",
|
222
|
+
examples=["source_railz.components.MyCustomPartitionRouter"],
|
223
|
+
title="Class Name",
|
224
224
|
)
|
225
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
225
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
226
226
|
|
227
227
|
|
228
228
|
class CustomTransformation(BaseModel):
|
229
229
|
class Config:
|
230
230
|
extra = Extra.allow
|
231
231
|
|
232
|
-
type: Literal[
|
232
|
+
type: Literal["CustomTransformation"]
|
233
233
|
class_name: str = Field(
|
234
234
|
...,
|
235
|
-
description=
|
236
|
-
examples=[
|
237
|
-
title=
|
235
|
+
description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_<name>.<package>.<class_name>`.",
|
236
|
+
examples=["source_railz.components.MyCustomTransformation"],
|
237
|
+
title="Class Name",
|
238
238
|
)
|
239
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
239
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
240
240
|
|
241
241
|
|
242
242
|
class RefreshTokenUpdater(BaseModel):
|
243
243
|
refresh_token_name: Optional[str] = Field(
|
244
|
-
|
245
|
-
description=
|
246
|
-
examples=[
|
247
|
-
title=
|
244
|
+
"refresh_token",
|
245
|
+
description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.",
|
246
|
+
examples=["refresh_token"],
|
247
|
+
title="Refresh Token Property Name",
|
248
248
|
)
|
249
249
|
access_token_config_path: Optional[List[str]] = Field(
|
250
|
-
[
|
251
|
-
description=
|
252
|
-
examples=[[
|
253
|
-
title=
|
250
|
+
["credentials", "access_token"],
|
251
|
+
description="Config path to the access token. Make sure the field actually exists in the config.",
|
252
|
+
examples=[["credentials", "access_token"], ["access_token"]],
|
253
|
+
title="Config Path To Access Token",
|
254
254
|
)
|
255
255
|
refresh_token_config_path: Optional[List[str]] = Field(
|
256
|
-
[
|
257
|
-
description=
|
258
|
-
examples=[[
|
259
|
-
title=
|
256
|
+
["credentials", "refresh_token"],
|
257
|
+
description="Config path to the access token. Make sure the field actually exists in the config.",
|
258
|
+
examples=[["credentials", "refresh_token"], ["refresh_token"]],
|
259
|
+
title="Config Path To Refresh Token",
|
260
260
|
)
|
261
261
|
token_expiry_date_config_path: Optional[List[str]] = Field(
|
262
|
-
[
|
263
|
-
description=
|
264
|
-
examples=[[
|
265
|
-
title=
|
262
|
+
["credentials", "token_expiry_date"],
|
263
|
+
description="Config path to the expiry date. Make sure actually exists in the config.",
|
264
|
+
examples=[["credentials", "token_expiry_date"]],
|
265
|
+
title="Config Path To Expiry Date",
|
266
266
|
)
|
267
267
|
|
268
268
|
|
269
269
|
class OAuthAuthenticator(BaseModel):
|
270
|
-
type: Literal[
|
270
|
+
type: Literal["OAuthAuthenticator"]
|
271
271
|
client_id: str = Field(
|
272
272
|
...,
|
273
|
-
description=
|
273
|
+
description="The OAuth client ID. Fill it in the user inputs.",
|
274
274
|
examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"],
|
275
|
-
title=
|
275
|
+
title="Client ID",
|
276
276
|
)
|
277
277
|
client_secret: str = Field(
|
278
278
|
...,
|
279
|
-
description=
|
279
|
+
description="The OAuth client secret. Fill it in the user inputs.",
|
280
280
|
examples=[
|
281
281
|
"{{ config['client_secret }}",
|
282
282
|
"{{ config['credentials']['client_secret }}",
|
283
283
|
],
|
284
|
-
title=
|
284
|
+
title="Client Secret",
|
285
285
|
)
|
286
286
|
refresh_token: Optional[str] = Field(
|
287
287
|
None,
|
288
|
-
description=
|
288
|
+
description="Credential artifact used to get a new access token.",
|
289
289
|
examples=[
|
290
290
|
"{{ config['refresh_token'] }}",
|
291
291
|
"{{ config['credentials]['refresh_token'] }}",
|
292
292
|
],
|
293
|
-
title=
|
293
|
+
title="Refresh Token",
|
294
294
|
)
|
295
295
|
token_refresh_endpoint: str = Field(
|
296
296
|
...,
|
297
|
-
description=
|
298
|
-
examples=[
|
299
|
-
title=
|
297
|
+
description="The full URL to call to obtain a new access token.",
|
298
|
+
examples=["https://connect.squareup.com/oauth2/token"],
|
299
|
+
title="Token Refresh Endpoint",
|
300
300
|
)
|
301
301
|
access_token_name: Optional[str] = Field(
|
302
|
-
|
303
|
-
description=
|
304
|
-
examples=[
|
305
|
-
title=
|
302
|
+
"access_token",
|
303
|
+
description="The name of the property which contains the access token in the response from the token refresh endpoint.",
|
304
|
+
examples=["access_token"],
|
305
|
+
title="Access Token Property Name",
|
306
306
|
)
|
307
307
|
expires_in_name: Optional[str] = Field(
|
308
|
-
|
309
|
-
description=
|
310
|
-
examples=[
|
311
|
-
title=
|
308
|
+
"expires_in",
|
309
|
+
description="The name of the property which contains the expiry date in the response from the token refresh endpoint.",
|
310
|
+
examples=["expires_in"],
|
311
|
+
title="Token Expiry Property Name",
|
312
312
|
)
|
313
313
|
grant_type: Optional[str] = Field(
|
314
|
-
|
315
|
-
description=
|
316
|
-
examples=[
|
317
|
-
title=
|
314
|
+
"refresh_token",
|
315
|
+
description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.",
|
316
|
+
examples=["refresh_token", "client_credentials"],
|
317
|
+
title="Grant Type",
|
318
318
|
)
|
319
319
|
refresh_request_body: Optional[Dict[str, Any]] = Field(
|
320
320
|
None,
|
321
|
-
description=
|
321
|
+
description="Body of the request sent to get a new access token.",
|
322
322
|
examples=[
|
323
323
|
{
|
324
|
-
|
325
|
-
|
326
|
-
|
324
|
+
"applicationId": "{{ config['application_id'] }}",
|
325
|
+
"applicationSecret": "{{ config['application_secret'] }}",
|
326
|
+
"token": "{{ config['token'] }}",
|
327
327
|
}
|
328
328
|
],
|
329
|
-
title=
|
329
|
+
title="Refresh Request Body",
|
330
330
|
)
|
331
331
|
scopes: Optional[List[str]] = Field(
|
332
332
|
None,
|
333
|
-
description=
|
334
|
-
examples=[
|
335
|
-
|
336
|
-
],
|
337
|
-
title='Scopes',
|
333
|
+
description="List of scopes that should be granted to the access token.",
|
334
|
+
examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]],
|
335
|
+
title="Scopes",
|
338
336
|
)
|
339
337
|
token_expiry_date: Optional[str] = Field(
|
340
338
|
None,
|
341
|
-
description=
|
342
|
-
examples=[
|
343
|
-
title=
|
339
|
+
description="The access token expiry date.",
|
340
|
+
examples=["2023-04-06T07:12:10.421833+00:00", 1680842386],
|
341
|
+
title="Token Expiry Date",
|
344
342
|
)
|
345
343
|
token_expiry_date_format: Optional[str] = Field(
|
346
344
|
None,
|
347
|
-
description=
|
348
|
-
examples=[
|
349
|
-
title=
|
345
|
+
description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.",
|
346
|
+
examples=["%Y-%m-%d %H:%M:%S.%f+00:00"],
|
347
|
+
title="Token Expiry Date Format",
|
350
348
|
)
|
351
349
|
refresh_token_updater: Optional[RefreshTokenUpdater] = Field(
|
352
350
|
None,
|
353
|
-
description=
|
354
|
-
title=
|
351
|
+
description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.",
|
352
|
+
title="Token Updater",
|
355
353
|
)
|
356
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
354
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
357
355
|
|
358
356
|
|
359
357
|
class ExponentialBackoffStrategy(BaseModel):
|
360
|
-
type: Literal[
|
358
|
+
type: Literal["ExponentialBackoffStrategy"]
|
361
359
|
factor: Optional[Union[float, str]] = Field(
|
362
360
|
5,
|
363
|
-
description=
|
364
|
-
examples=[5, 5.5,
|
365
|
-
title=
|
361
|
+
description="Multiplicative constant applied on each retry.",
|
362
|
+
examples=[5, 5.5, "10"],
|
363
|
+
title="Factor",
|
366
364
|
)
|
367
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
365
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
368
366
|
|
369
367
|
|
370
368
|
class SessionTokenRequestBearerAuthenticator(BaseModel):
|
371
|
-
type: Literal[
|
369
|
+
type: Literal["Bearer"]
|
372
370
|
|
373
371
|
|
374
372
|
class HttpMethodEnum(Enum):
|
375
|
-
GET =
|
376
|
-
POST =
|
373
|
+
GET = "GET"
|
374
|
+
POST = "POST"
|
377
375
|
|
378
376
|
|
379
377
|
class Action(Enum):
|
380
|
-
SUCCESS =
|
381
|
-
FAIL =
|
382
|
-
RETRY =
|
383
|
-
IGNORE =
|
378
|
+
SUCCESS = "SUCCESS"
|
379
|
+
FAIL = "FAIL"
|
380
|
+
RETRY = "RETRY"
|
381
|
+
IGNORE = "IGNORE"
|
384
382
|
|
385
383
|
|
386
384
|
class HttpResponseFilter(BaseModel):
|
387
|
-
type: Literal[
|
385
|
+
type: Literal["HttpResponseFilter"]
|
388
386
|
action: Action = Field(
|
389
387
|
...,
|
390
|
-
description=
|
391
|
-
examples=[
|
392
|
-
title=
|
388
|
+
description="Action to execute if a response matches the filter.",
|
389
|
+
examples=["SUCCESS", "FAIL", "RETRY", "IGNORE"],
|
390
|
+
title="Action",
|
393
391
|
)
|
394
392
|
error_message: Optional[str] = Field(
|
395
393
|
None,
|
396
|
-
description=
|
397
|
-
title=
|
394
|
+
description="Error Message to display if the response matches the filter.",
|
395
|
+
title="Error Message",
|
398
396
|
)
|
399
397
|
error_message_contains: Optional[str] = Field(
|
400
398
|
None,
|
401
|
-
description=
|
402
|
-
example=[
|
403
|
-
title=
|
399
|
+
description="Match the response if its error message contains the substring.",
|
400
|
+
example=["This API operation is not enabled for this site"],
|
401
|
+
title="Error Message Substring",
|
404
402
|
)
|
405
403
|
http_codes: Optional[List[int]] = Field(
|
406
404
|
None,
|
407
|
-
description=
|
405
|
+
description="Match the response if its HTTP code is included in this list.",
|
408
406
|
examples=[[420, 429], [500]],
|
409
|
-
title=
|
407
|
+
title="HTTP Codes",
|
410
408
|
)
|
411
409
|
predicate: Optional[str] = Field(
|
412
410
|
None,
|
413
|
-
description=
|
411
|
+
description="Match the response if the predicate evaluates to true.",
|
414
412
|
examples=[
|
415
413
|
"{{ 'Too much requests' in response }}",
|
416
414
|
"{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}",
|
417
415
|
],
|
418
|
-
title=
|
416
|
+
title="Predicate",
|
419
417
|
)
|
420
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
418
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
421
419
|
|
422
420
|
|
423
421
|
class InlineSchemaLoader(BaseModel):
|
424
|
-
type: Literal[
|
422
|
+
type: Literal["InlineSchemaLoader"]
|
425
423
|
schema_: Optional[Dict[str, Any]] = Field(
|
426
424
|
None,
|
427
|
-
alias=
|
425
|
+
alias="schema",
|
428
426
|
description='Describes a streams\' schema. Refer to the <a href="https://docs.airbyte.com/understanding-airbyte/supported-data-types/">Data Types documentation</a> for more details on which types are valid.',
|
429
|
-
title=
|
427
|
+
title="Schema",
|
430
428
|
)
|
431
429
|
|
432
430
|
|
433
431
|
class JsonFileSchemaLoader(BaseModel):
|
434
|
-
type: Literal[
|
432
|
+
type: Literal["JsonFileSchemaLoader"]
|
435
433
|
file_path: Optional[str] = Field(
|
436
434
|
None,
|
437
435
|
description="Path to the JSON file defining the schema. The path is relative to the connector module's root.",
|
438
|
-
example=[
|
439
|
-
title=
|
436
|
+
example=["./schemas/users.json"],
|
437
|
+
title="File Path",
|
440
438
|
)
|
441
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
439
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
442
440
|
|
443
441
|
|
444
442
|
class JsonDecoder(BaseModel):
|
445
|
-
type: Literal[
|
443
|
+
type: Literal["JsonDecoder"]
|
446
444
|
|
447
445
|
|
448
446
|
class MinMaxDatetime(BaseModel):
|
449
|
-
type: Literal[
|
447
|
+
type: Literal["MinMaxDatetime"]
|
450
448
|
datetime: str = Field(
|
451
449
|
...,
|
452
|
-
description=
|
453
|
-
examples=[
|
454
|
-
title=
|
450
|
+
description="Datetime value.",
|
451
|
+
examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"],
|
452
|
+
title="Datetime",
|
455
453
|
)
|
456
454
|
datetime_format: Optional[str] = Field(
|
457
|
-
|
455
|
+
"",
|
458
456
|
description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n',
|
459
|
-
examples=[
|
460
|
-
title=
|
457
|
+
examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"],
|
458
|
+
title="Datetime Format",
|
461
459
|
)
|
462
460
|
max_datetime: Optional[str] = Field(
|
463
461
|
None,
|
464
|
-
description=
|
465
|
-
examples=[
|
466
|
-
title=
|
462
|
+
description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.",
|
463
|
+
examples=["2021-01-01T00:00:00Z", "2021-01-01"],
|
464
|
+
title="Max Datetime",
|
467
465
|
)
|
468
466
|
min_datetime: Optional[str] = Field(
|
469
467
|
None,
|
470
|
-
description=
|
471
|
-
examples=[
|
472
|
-
title=
|
468
|
+
description="Floor applied on the datetime value. Must be formatted with the datetime_format field.",
|
469
|
+
examples=["2010-01-01T00:00:00Z", "2010-01-01"],
|
470
|
+
title="Min Datetime",
|
473
471
|
)
|
474
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
472
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
475
473
|
|
476
474
|
|
477
475
|
class NoAuth(BaseModel):
|
478
|
-
type: Literal[
|
479
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
476
|
+
type: Literal["NoAuth"]
|
477
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
480
478
|
|
481
479
|
|
482
480
|
class NoPagination(BaseModel):
|
483
|
-
type: Literal[
|
481
|
+
type: Literal["NoPagination"]
|
484
482
|
|
485
483
|
|
486
484
|
class OAuthConfigSpecification(BaseModel):
|
487
485
|
class Config:
|
488
486
|
extra = Extra.allow
|
489
487
|
|
490
|
-
oauth_user_input_from_connector_config_specification: Optional[
|
491
|
-
Dict[str, Any]
|
492
|
-
] = Field(
|
488
|
+
oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field(
|
493
489
|
None,
|
494
490
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }",
|
495
491
|
examples=[
|
496
|
-
{
|
492
|
+
{"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}},
|
497
493
|
{
|
498
|
-
|
499
|
-
|
500
|
-
|
494
|
+
"app_id": {
|
495
|
+
"type": "string",
|
496
|
+
"path_in_connector_config": ["info", "app_id"],
|
501
497
|
}
|
502
498
|
},
|
503
499
|
],
|
504
|
-
title=
|
500
|
+
title="OAuth user input",
|
505
501
|
)
|
506
502
|
complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
|
507
503
|
None,
|
508
504
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
|
509
505
|
examples=[
|
510
506
|
{
|
511
|
-
|
512
|
-
|
513
|
-
|
507
|
+
"refresh_token": {
|
508
|
+
"type": "string,",
|
509
|
+
"path_in_connector_config": ["credentials", "refresh_token"],
|
514
510
|
}
|
515
511
|
}
|
516
512
|
],
|
517
|
-
title=
|
513
|
+
title="OAuth output specification",
|
518
514
|
)
|
519
515
|
complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field(
|
520
516
|
None,
|
521
|
-
description=
|
522
|
-
examples=[
|
523
|
-
|
524
|
-
],
|
525
|
-
title='OAuth input specification',
|
517
|
+
description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }",
|
518
|
+
examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}],
|
519
|
+
title="OAuth input specification",
|
526
520
|
)
|
527
521
|
complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field(
|
528
522
|
None,
|
529
523
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }",
|
530
524
|
examples=[
|
531
525
|
{
|
532
|
-
|
533
|
-
|
534
|
-
|
526
|
+
"client_id": {
|
527
|
+
"type": "string,",
|
528
|
+
"path_in_connector_config": ["credentials", "client_id"],
|
535
529
|
},
|
536
|
-
|
537
|
-
|
538
|
-
|
530
|
+
"client_secret": {
|
531
|
+
"type": "string,",
|
532
|
+
"path_in_connector_config": ["credentials", "client_secret"],
|
539
533
|
},
|
540
534
|
}
|
541
535
|
],
|
542
|
-
title=
|
536
|
+
title="OAuth server output specification",
|
543
537
|
)
|
544
538
|
|
545
539
|
|
546
540
|
class OffsetIncrement(BaseModel):
|
547
|
-
type: Literal[
|
541
|
+
type: Literal["OffsetIncrement"]
|
548
542
|
page_size: Optional[Union[int, str]] = Field(
|
549
543
|
None,
|
550
|
-
description=
|
544
|
+
description="The number of records to include in each pages.",
|
551
545
|
examples=[100, "{{ config['page_size'] }}"],
|
552
|
-
title=
|
546
|
+
title="Limit",
|
553
547
|
)
|
554
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
548
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
555
549
|
|
556
550
|
|
557
551
|
class PageIncrement(BaseModel):
|
558
|
-
type: Literal[
|
552
|
+
type: Literal["PageIncrement"]
|
559
553
|
page_size: Optional[int] = Field(
|
560
554
|
None,
|
561
|
-
description=
|
562
|
-
examples=[100,
|
563
|
-
title=
|
555
|
+
description="The number of records to include in each pages.",
|
556
|
+
examples=[100, "100"],
|
557
|
+
title="Page Size",
|
564
558
|
)
|
565
559
|
start_from_page: Optional[int] = Field(
|
566
560
|
0,
|
567
|
-
description=
|
561
|
+
description="Index of the first page to request.",
|
568
562
|
examples=[0, 1],
|
569
|
-
title=
|
563
|
+
title="Start From Page",
|
570
564
|
)
|
571
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
565
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
572
566
|
|
573
567
|
|
574
568
|
class PrimaryKey(BaseModel):
|
575
569
|
__root__: Union[str, List[str], List[List[str]]] = Field(
|
576
570
|
...,
|
577
|
-
description=
|
578
|
-
examples=[
|
579
|
-
title=
|
571
|
+
description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.",
|
572
|
+
examples=["id", ["code", "type"]],
|
573
|
+
title="Primary Key",
|
580
574
|
)
|
581
575
|
|
582
576
|
|
583
577
|
class RecordFilter(BaseModel):
|
584
|
-
type: Literal[
|
578
|
+
type: Literal["RecordFilter"]
|
585
579
|
condition: Optional[str] = Field(
|
586
|
-
|
587
|
-
description=
|
580
|
+
"",
|
581
|
+
description="The predicate to filter a record. Records will be removed if evaluated to False.",
|
588
582
|
examples=[
|
589
583
|
"{{ record['created_at'] >= stream_interval['start_time'] }}",
|
590
584
|
"{{ record.status in ['active', 'expired'] }}",
|
591
585
|
],
|
592
586
|
)
|
593
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
587
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
594
588
|
|
595
589
|
|
596
590
|
class RemoveFields(BaseModel):
|
597
|
-
type: Literal[
|
591
|
+
type: Literal["RemoveFields"]
|
598
592
|
field_pointers: List[List[str]] = Field(
|
599
593
|
...,
|
600
|
-
description=
|
601
|
-
examples=[[
|
602
|
-
title=
|
594
|
+
description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.",
|
595
|
+
examples=[["tags"], [["content", "html"], ["content", "plain_text"]]],
|
596
|
+
title="Field Paths",
|
603
597
|
)
|
604
598
|
|
605
599
|
|
606
600
|
class RequestPath(BaseModel):
|
607
|
-
type: Literal[
|
601
|
+
type: Literal["RequestPath"]
|
608
602
|
|
609
603
|
|
610
604
|
class InjectInto(Enum):
|
611
|
-
request_parameter =
|
612
|
-
header =
|
613
|
-
body_data =
|
614
|
-
body_json =
|
605
|
+
request_parameter = "request_parameter"
|
606
|
+
header = "header"
|
607
|
+
body_data = "body_data"
|
608
|
+
body_json = "body_json"
|
615
609
|
|
616
610
|
|
617
611
|
class RequestOption(BaseModel):
|
618
|
-
type: Literal[
|
612
|
+
type: Literal["RequestOption"]
|
619
613
|
field_name: str = Field(
|
620
614
|
...,
|
621
|
-
description=
|
622
|
-
examples=[
|
623
|
-
title=
|
615
|
+
description="Configures which key should be used in the location that the descriptor is being injected into",
|
616
|
+
examples=["segment_id"],
|
617
|
+
title="Request Option",
|
624
618
|
)
|
625
619
|
inject_into: InjectInto = Field(
|
626
620
|
...,
|
627
|
-
description=
|
628
|
-
examples=[
|
629
|
-
title=
|
621
|
+
description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.",
|
622
|
+
examples=["request_parameter", "header", "body_data", "body_json"],
|
623
|
+
title="Inject Into",
|
630
624
|
)
|
631
625
|
|
632
626
|
|
@@ -638,253 +632,251 @@ class Schemas(BaseModel):
|
|
638
632
|
|
639
633
|
|
640
634
|
class LegacySessionTokenAuthenticator(BaseModel):
|
641
|
-
type: Literal[
|
635
|
+
type: Literal["LegacySessionTokenAuthenticator"]
|
642
636
|
header: str = Field(
|
643
637
|
...,
|
644
|
-
description=
|
645
|
-
examples=[
|
646
|
-
title=
|
638
|
+
description="The name of the session token header that will be injected in the request",
|
639
|
+
examples=["X-Session"],
|
640
|
+
title="Session Request Header",
|
647
641
|
)
|
648
642
|
login_url: str = Field(
|
649
643
|
...,
|
650
|
-
description=
|
651
|
-
examples=[
|
652
|
-
title=
|
644
|
+
description="Path of the login URL (do not include the base URL)",
|
645
|
+
examples=["session"],
|
646
|
+
title="Login Path",
|
653
647
|
)
|
654
648
|
session_token: Optional[str] = Field(
|
655
649
|
None,
|
656
|
-
description=
|
650
|
+
description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair",
|
657
651
|
example=["{{ config['session_token'] }}"],
|
658
|
-
title=
|
652
|
+
title="Session Token",
|
659
653
|
)
|
660
654
|
session_token_response_key: str = Field(
|
661
655
|
...,
|
662
|
-
description=
|
663
|
-
examples=[
|
664
|
-
title=
|
656
|
+
description="Name of the key of the session token to be extracted from the response",
|
657
|
+
examples=["id"],
|
658
|
+
title="Response Token Response Key",
|
665
659
|
)
|
666
660
|
username: Optional[str] = Field(
|
667
661
|
None,
|
668
|
-
description=
|
662
|
+
description="Username used to authenticate and obtain a session token",
|
669
663
|
examples=[" {{ config['username'] }}"],
|
670
|
-
title=
|
664
|
+
title="Username",
|
671
665
|
)
|
672
666
|
password: Optional[str] = Field(
|
673
|
-
|
674
|
-
description=
|
675
|
-
examples=["{{ config['password'] }}",
|
676
|
-
title=
|
667
|
+
"",
|
668
|
+
description="Password used to authenticate and obtain a session token",
|
669
|
+
examples=["{{ config['password'] }}", ""],
|
670
|
+
title="Password",
|
677
671
|
)
|
678
672
|
validate_session_url: str = Field(
|
679
673
|
...,
|
680
|
-
description=
|
681
|
-
examples=[
|
682
|
-
title=
|
674
|
+
description="Path of the URL to use to validate that the session token is valid (do not include the base URL)",
|
675
|
+
examples=["user/current"],
|
676
|
+
title="Validate Session Path",
|
683
677
|
)
|
684
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
678
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
685
679
|
|
686
680
|
|
687
681
|
class WaitTimeFromHeader(BaseModel):
|
688
|
-
type: Literal[
|
682
|
+
type: Literal["WaitTimeFromHeader"]
|
689
683
|
header: str = Field(
|
690
684
|
...,
|
691
|
-
description=
|
692
|
-
examples=[
|
693
|
-
title=
|
685
|
+
description="The name of the response header defining how long to wait before retrying.",
|
686
|
+
examples=["Retry-After"],
|
687
|
+
title="Response Header Name",
|
694
688
|
)
|
695
689
|
regex: Optional[str] = Field(
|
696
690
|
None,
|
697
|
-
description=
|
698
|
-
examples=[
|
699
|
-
title=
|
691
|
+
description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
|
692
|
+
examples=["([-+]?\\d+)"],
|
693
|
+
title="Extraction Regex",
|
700
694
|
)
|
701
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
695
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
702
696
|
|
703
697
|
|
704
698
|
class WaitUntilTimeFromHeader(BaseModel):
|
705
|
-
type: Literal[
|
699
|
+
type: Literal["WaitUntilTimeFromHeader"]
|
706
700
|
header: str = Field(
|
707
701
|
...,
|
708
|
-
description=
|
709
|
-
examples=[
|
710
|
-
title=
|
702
|
+
description="The name of the response header defining how long to wait before retrying.",
|
703
|
+
examples=["wait_time"],
|
704
|
+
title="Response Header",
|
711
705
|
)
|
712
706
|
min_wait: Optional[Union[float, str]] = Field(
|
713
707
|
None,
|
714
|
-
description=
|
715
|
-
examples=[10,
|
716
|
-
title=
|
708
|
+
description="Minimum time to wait before retrying.",
|
709
|
+
examples=[10, "60"],
|
710
|
+
title="Minimum Wait Time",
|
717
711
|
)
|
718
712
|
regex: Optional[str] = Field(
|
719
713
|
None,
|
720
|
-
description=
|
721
|
-
examples=[
|
722
|
-
title=
|
714
|
+
description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.",
|
715
|
+
examples=["([-+]?\\d+)"],
|
716
|
+
title="Extraction Regex",
|
723
717
|
)
|
724
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
718
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
725
719
|
|
726
720
|
|
727
721
|
class ApiKeyAuthenticator(BaseModel):
|
728
|
-
type: Literal[
|
722
|
+
type: Literal["ApiKeyAuthenticator"]
|
729
723
|
api_token: Optional[str] = Field(
|
730
724
|
None,
|
731
|
-
description=
|
725
|
+
description="The API key to inject in the request. Fill it in the user inputs.",
|
732
726
|
examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"],
|
733
|
-
title=
|
727
|
+
title="API Key",
|
734
728
|
)
|
735
729
|
header: Optional[str] = Field(
|
736
730
|
None,
|
737
|
-
description=
|
738
|
-
examples=[
|
739
|
-
title=
|
731
|
+
description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.",
|
732
|
+
examples=["Authorization", "Api-Token", "X-Auth-Token"],
|
733
|
+
title="Header Name",
|
740
734
|
)
|
741
735
|
inject_into: Optional[RequestOption] = Field(
|
742
736
|
None,
|
743
|
-
description=
|
737
|
+
description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.",
|
744
738
|
examples=[
|
745
|
-
{
|
746
|
-
{
|
739
|
+
{"inject_into": "header", "field_name": "Authorization"},
|
740
|
+
{"inject_into": "request_parameter", "field_name": "authKey"},
|
747
741
|
],
|
748
|
-
title=
|
742
|
+
title="Inject API Key Into Outgoing HTTP Request",
|
749
743
|
)
|
750
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
744
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
751
745
|
|
752
746
|
|
753
747
|
class AuthFlow(BaseModel):
|
754
|
-
auth_flow_type: Optional[AuthFlowType] = Field(
|
755
|
-
None, description='The type of auth to use', title='Auth flow type'
|
756
|
-
)
|
748
|
+
auth_flow_type: Optional[AuthFlowType] = Field(None, description="The type of auth to use", title="Auth flow type")
|
757
749
|
predicate_key: Optional[List[str]] = Field(
|
758
750
|
None,
|
759
|
-
description=
|
760
|
-
examples=[[
|
761
|
-
title=
|
751
|
+
description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.",
|
752
|
+
examples=[["credentials", "auth_type"]],
|
753
|
+
title="Predicate key",
|
762
754
|
)
|
763
755
|
predicate_value: Optional[str] = Field(
|
764
756
|
None,
|
765
|
-
description=
|
766
|
-
examples=[
|
767
|
-
title=
|
757
|
+
description="Value of the predicate_key fields for the advanced auth to be applicable.",
|
758
|
+
examples=["Oauth"],
|
759
|
+
title="Predicate value",
|
768
760
|
)
|
769
761
|
oauth_config_specification: Optional[OAuthConfigSpecification] = None
|
770
762
|
|
771
763
|
|
772
764
|
class CursorPagination(BaseModel):
|
773
|
-
type: Literal[
|
765
|
+
type: Literal["CursorPagination"]
|
774
766
|
cursor_value: str = Field(
|
775
767
|
...,
|
776
|
-
description=
|
768
|
+
description="Value of the cursor defining the next page to fetch.",
|
777
769
|
examples=[
|
778
|
-
|
770
|
+
"{{ headers.link.next.cursor }}",
|
779
771
|
"{{ last_records[-1]['key'] }}",
|
780
772
|
"{{ response['nextPage'] }}",
|
781
773
|
],
|
782
|
-
title=
|
774
|
+
title="Cursor Value",
|
783
775
|
)
|
784
776
|
page_size: Optional[int] = Field(
|
785
777
|
None,
|
786
|
-
description=
|
778
|
+
description="The number of records to include in each pages.",
|
787
779
|
examples=[100],
|
788
|
-
title=
|
780
|
+
title="Page Size",
|
789
781
|
)
|
790
782
|
stop_condition: Optional[str] = Field(
|
791
783
|
None,
|
792
|
-
description=
|
784
|
+
description="Template string evaluating when to stop paginating.",
|
793
785
|
examples=[
|
794
|
-
|
786
|
+
"{{ response.data.has_more is false }}",
|
795
787
|
"{{ 'next' not in headers['link'] }}",
|
796
788
|
],
|
797
|
-
title=
|
789
|
+
title="Stop Condition",
|
798
790
|
)
|
799
791
|
decoder: Optional[JsonDecoder] = Field(
|
800
792
|
None,
|
801
|
-
description=
|
802
|
-
title=
|
793
|
+
description="Component decoding the response so records can be extracted.",
|
794
|
+
title="Decoder",
|
803
795
|
)
|
804
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
796
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
805
797
|
|
806
798
|
|
807
799
|
class DatetimeBasedCursor(BaseModel):
|
808
|
-
type: Literal[
|
800
|
+
type: Literal["DatetimeBasedCursor"]
|
809
801
|
cursor_field: str = Field(
|
810
802
|
...,
|
811
|
-
description=
|
812
|
-
examples=[
|
813
|
-
title=
|
803
|
+
description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.",
|
804
|
+
examples=["created_at", "{{ config['record_cursor'] }}"],
|
805
|
+
title="Cursor Field",
|
814
806
|
)
|
815
807
|
datetime_format: str = Field(
|
816
808
|
...,
|
817
|
-
description=
|
818
|
-
examples=[
|
819
|
-
title=
|
809
|
+
description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n",
|
810
|
+
examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms"],
|
811
|
+
title="Outgoing Datetime Format",
|
820
812
|
)
|
821
813
|
start_datetime: Union[str, MinMaxDatetime] = Field(
|
822
814
|
...,
|
823
|
-
description=
|
824
|
-
examples=[
|
825
|
-
title=
|
815
|
+
description="The datetime that determines the earliest record that should be synced.",
|
816
|
+
examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"],
|
817
|
+
title="Start Datetime",
|
826
818
|
)
|
827
819
|
cursor_datetime_formats: Optional[List[str]] = Field(
|
828
820
|
None,
|
829
|
-
description=
|
830
|
-
title=
|
821
|
+
description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.",
|
822
|
+
title="Cursor Datetime Formats",
|
831
823
|
)
|
832
824
|
cursor_granularity: Optional[str] = Field(
|
833
825
|
None,
|
834
|
-
description=
|
835
|
-
examples=[
|
836
|
-
title=
|
826
|
+
description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.",
|
827
|
+
examples=["PT1S"],
|
828
|
+
title="Cursor Granularity",
|
837
829
|
)
|
838
830
|
end_datetime: Optional[Union[str, MinMaxDatetime]] = Field(
|
839
831
|
None,
|
840
|
-
description=
|
841
|
-
examples=[
|
842
|
-
title=
|
832
|
+
description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.",
|
833
|
+
examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"],
|
834
|
+
title="End Datetime",
|
843
835
|
)
|
844
836
|
end_time_option: Optional[RequestOption] = Field(
|
845
837
|
None,
|
846
|
-
description=
|
847
|
-
title=
|
838
|
+
description="Optionally configures how the end datetime will be sent in requests to the source API.",
|
839
|
+
title="Inject End Time Into Outgoing HTTP Request",
|
848
840
|
)
|
849
841
|
is_data_feed: Optional[bool] = Field(
|
850
842
|
None,
|
851
|
-
description=
|
852
|
-
title=
|
843
|
+
description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.",
|
844
|
+
title="Whether the target API is formatted as a data feed",
|
853
845
|
)
|
854
846
|
lookback_window: Optional[str] = Field(
|
855
847
|
None,
|
856
|
-
description=
|
857
|
-
examples=[
|
858
|
-
title=
|
848
|
+
description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.",
|
849
|
+
examples=["P1D", "P{{ config['lookback_days'] }}D"],
|
850
|
+
title="Lookback Window",
|
859
851
|
)
|
860
852
|
partition_field_end: Optional[str] = Field(
|
861
853
|
None,
|
862
|
-
description=
|
863
|
-
examples=[
|
864
|
-
title=
|
854
|
+
description="Name of the partition start time field.",
|
855
|
+
examples=["ending_time"],
|
856
|
+
title="Partition Field End",
|
865
857
|
)
|
866
858
|
partition_field_start: Optional[str] = Field(
|
867
859
|
None,
|
868
|
-
description=
|
869
|
-
examples=[
|
870
|
-
title=
|
860
|
+
description="Name of the partition end time field.",
|
861
|
+
examples=["starting_time"],
|
862
|
+
title="Partition Field Start",
|
871
863
|
)
|
872
864
|
start_time_option: Optional[RequestOption] = Field(
|
873
865
|
None,
|
874
|
-
description=
|
875
|
-
title=
|
866
|
+
description="Optionally configures how the start datetime will be sent in requests to the source API.",
|
867
|
+
title="Inject Start Time Into Outgoing HTTP Request",
|
876
868
|
)
|
877
869
|
step: Optional[str] = Field(
|
878
870
|
None,
|
879
|
-
description=
|
880
|
-
examples=[
|
881
|
-
title=
|
871
|
+
description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.",
|
872
|
+
examples=["P1W", "{{ config['step_increment'] }}"],
|
873
|
+
title="Step",
|
882
874
|
)
|
883
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
875
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
884
876
|
|
885
877
|
|
886
878
|
class DefaultErrorHandler(BaseModel):
|
887
|
-
type: Literal[
|
879
|
+
type: Literal["DefaultErrorHandler"]
|
888
880
|
backoff_strategies: Optional[
|
889
881
|
List[
|
890
882
|
Union[
|
@@ -897,144 +889,142 @@ class DefaultErrorHandler(BaseModel):
|
|
897
889
|
]
|
898
890
|
] = Field(
|
899
891
|
None,
|
900
|
-
description=
|
901
|
-
title=
|
892
|
+
description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.",
|
893
|
+
title="Backoff Strategies",
|
902
894
|
)
|
903
895
|
max_retries: Optional[int] = Field(
|
904
896
|
5,
|
905
|
-
description=
|
897
|
+
description="The maximum number of time to retry a retryable request before giving up and failing.",
|
906
898
|
examples=[5, 0, 10],
|
907
|
-
title=
|
899
|
+
title="Max Retry Count",
|
908
900
|
)
|
909
901
|
response_filters: Optional[List[HttpResponseFilter]] = Field(
|
910
902
|
None,
|
911
903
|
description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.",
|
912
|
-
title=
|
904
|
+
title="Response Filters",
|
913
905
|
)
|
914
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
906
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
915
907
|
|
916
908
|
|
917
909
|
class DefaultPaginator(BaseModel):
|
918
|
-
type: Literal[
|
919
|
-
pagination_strategy: Union[
|
920
|
-
CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement
|
921
|
-
] = Field(
|
910
|
+
type: Literal["DefaultPaginator"]
|
911
|
+
pagination_strategy: Union[CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement] = Field(
|
922
912
|
...,
|
923
|
-
description=
|
924
|
-
title=
|
913
|
+
description="Strategy defining how records are paginated.",
|
914
|
+
title="Pagination Strategy",
|
925
915
|
)
|
926
916
|
decoder: Optional[JsonDecoder] = Field(
|
927
917
|
None,
|
928
|
-
description=
|
929
|
-
title=
|
918
|
+
description="Component decoding the response so records can be extracted.",
|
919
|
+
title="Decoder",
|
930
920
|
)
|
931
921
|
page_size_option: Optional[RequestOption] = None
|
932
922
|
page_token_option: Optional[Union[RequestOption, RequestPath]] = None
|
933
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
923
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
934
924
|
|
935
925
|
|
936
926
|
class DpathExtractor(BaseModel):
|
937
|
-
type: Literal[
|
927
|
+
type: Literal["DpathExtractor"]
|
938
928
|
field_path: List[str] = Field(
|
939
929
|
...,
|
940
930
|
description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
|
941
931
|
examples=[
|
942
|
-
[
|
943
|
-
[
|
944
|
-
[
|
945
|
-
[
|
932
|
+
["data"],
|
933
|
+
["data", "records"],
|
934
|
+
["data", "{{ parameters.name }}"],
|
935
|
+
["data", "*", "record"],
|
946
936
|
],
|
947
|
-
title=
|
937
|
+
title="Field Path",
|
948
938
|
)
|
949
939
|
decoder: Optional[JsonDecoder] = Field(
|
950
940
|
None,
|
951
|
-
description=
|
952
|
-
title=
|
941
|
+
description="Component decoding the response so records can be extracted.",
|
942
|
+
title="Decoder",
|
953
943
|
)
|
954
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
944
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
955
945
|
|
956
946
|
|
957
947
|
class SessionTokenRequestApiKeyAuthenticator(BaseModel):
|
958
|
-
type: Literal[
|
948
|
+
type: Literal["ApiKey"]
|
959
949
|
inject_into: RequestOption = Field(
|
960
950
|
...,
|
961
|
-
description=
|
951
|
+
description="Configure how the API Key will be sent in requests to the source API.",
|
962
952
|
examples=[
|
963
|
-
{
|
964
|
-
{
|
953
|
+
{"inject_into": "header", "field_name": "Authorization"},
|
954
|
+
{"inject_into": "request_parameter", "field_name": "authKey"},
|
965
955
|
],
|
966
|
-
title=
|
956
|
+
title="Inject API Key Into Outgoing HTTP Request",
|
967
957
|
)
|
968
958
|
|
969
959
|
|
970
960
|
class ListPartitionRouter(BaseModel):
|
971
|
-
type: Literal[
|
961
|
+
type: Literal["ListPartitionRouter"]
|
972
962
|
cursor_field: str = Field(
|
973
963
|
...,
|
974
964
|
description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.',
|
975
|
-
examples=[
|
976
|
-
title=
|
965
|
+
examples=["section", "{{ config['section_key'] }}"],
|
966
|
+
title="Current Partition Value Identifier",
|
977
967
|
)
|
978
968
|
values: Union[str, List[str]] = Field(
|
979
969
|
...,
|
980
|
-
description=
|
981
|
-
examples=[[
|
982
|
-
title=
|
970
|
+
description="The list of attributes being iterated over and used as input for the requests made to the source API.",
|
971
|
+
examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"],
|
972
|
+
title="Partition Values",
|
983
973
|
)
|
984
974
|
request_option: Optional[RequestOption] = Field(
|
985
975
|
None,
|
986
|
-
description=
|
987
|
-
title=
|
976
|
+
description="A request option describing where the list value should be injected into and under what field name if applicable.",
|
977
|
+
title="Inject Partition Value Into Outgoing HTTP Request",
|
988
978
|
)
|
989
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
979
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
990
980
|
|
991
981
|
|
992
982
|
class RecordSelector(BaseModel):
|
993
|
-
type: Literal[
|
983
|
+
type: Literal["RecordSelector"]
|
994
984
|
extractor: Union[CustomRecordExtractor, DpathExtractor]
|
995
985
|
record_filter: Optional[RecordFilter] = Field(
|
996
986
|
None,
|
997
|
-
description=
|
998
|
-
title=
|
987
|
+
description="Responsible for filtering records to be emitted by the Source.",
|
988
|
+
title="Record Filter",
|
999
989
|
)
|
1000
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
990
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1001
991
|
|
1002
992
|
|
1003
993
|
class Spec(BaseModel):
|
1004
|
-
type: Literal[
|
994
|
+
type: Literal["Spec"]
|
1005
995
|
connection_specification: Dict[str, Any] = Field(
|
1006
996
|
...,
|
1007
|
-
description=
|
1008
|
-
title=
|
997
|
+
description="A connection specification describing how a the connector can be configured.",
|
998
|
+
title="Connection Specification",
|
1009
999
|
)
|
1010
1000
|
documentation_url: Optional[str] = Field(
|
1011
1001
|
None,
|
1012
1002
|
description="URL of the connector's documentation page.",
|
1013
|
-
examples=[
|
1014
|
-
title=
|
1003
|
+
examples=["https://docs.airbyte.com/integrations/sources/dremio"],
|
1004
|
+
title="Documentation URL",
|
1015
1005
|
)
|
1016
1006
|
advanced_auth: Optional[AuthFlow] = Field(
|
1017
1007
|
None,
|
1018
|
-
description=
|
1019
|
-
title=
|
1008
|
+
description="Advanced specification for configuring the authentication flow.",
|
1009
|
+
title="Advanced Auth",
|
1020
1010
|
)
|
1021
1011
|
|
1022
1012
|
|
1023
1013
|
class CompositeErrorHandler(BaseModel):
|
1024
|
-
type: Literal[
|
1014
|
+
type: Literal["CompositeErrorHandler"]
|
1025
1015
|
error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field(
|
1026
1016
|
...,
|
1027
|
-
description=
|
1028
|
-
title=
|
1017
|
+
description="List of error handlers to iterate on to determine how to handle a failed response.",
|
1018
|
+
title="Error Handlers",
|
1029
1019
|
)
|
1030
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1020
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1031
1021
|
|
1032
1022
|
|
1033
1023
|
class DeclarativeSource(BaseModel):
|
1034
1024
|
class Config:
|
1035
1025
|
extra = Extra.forbid
|
1036
1026
|
|
1037
|
-
type: Literal[
|
1027
|
+
type: Literal["DeclarativeSource"]
|
1038
1028
|
check: CheckStream
|
1039
1029
|
streams: List[DeclarativeStream]
|
1040
1030
|
version: str
|
@@ -1043,7 +1033,7 @@ class DeclarativeSource(BaseModel):
|
|
1043
1033
|
spec: Optional[Spec] = None
|
1044
1034
|
metadata: Optional[Dict[str, Any]] = Field(
|
1045
1035
|
None,
|
1046
|
-
description=
|
1036
|
+
description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
|
1047
1037
|
)
|
1048
1038
|
|
1049
1039
|
|
@@ -1051,101 +1041,91 @@ class DeclarativeStream(BaseModel):
|
|
1051
1041
|
class Config:
|
1052
1042
|
extra = Extra.allow
|
1053
1043
|
|
1054
|
-
type: Literal[
|
1044
|
+
type: Literal["DeclarativeStream"]
|
1055
1045
|
retriever: Union[CustomRetriever, SimpleRetriever] = Field(
|
1056
1046
|
...,
|
1057
|
-
description=
|
1058
|
-
title=
|
1047
|
+
description="Component used to coordinate how records are extracted across stream slices and request pages.",
|
1048
|
+
title="Retriever",
|
1059
1049
|
)
|
1060
|
-
incremental_sync: Optional[
|
1061
|
-
Union[CustomIncrementalSync, DatetimeBasedCursor]
|
1062
|
-
] = Field(
|
1050
|
+
incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field(
|
1063
1051
|
None,
|
1064
|
-
description=
|
1065
|
-
title=
|
1066
|
-
)
|
1067
|
-
name: Optional[str] = Field(
|
1068
|
-
'', description='The stream name.', example=['Users'], title='Name'
|
1069
|
-
)
|
1070
|
-
primary_key: Optional[PrimaryKey] = Field(
|
1071
|
-
'', description='The primary key of the stream.', title='Primary Key'
|
1052
|
+
description="Component used to fetch data incrementally based on a time field in the data.",
|
1053
|
+
title="Incremental Sync",
|
1072
1054
|
)
|
1055
|
+
name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name")
|
1056
|
+
primary_key: Optional[PrimaryKey] = Field("", description="The primary key of the stream.", title="Primary Key")
|
1073
1057
|
schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader]] = Field(
|
1074
1058
|
None,
|
1075
|
-
description=
|
1076
|
-
title=
|
1059
|
+
description="Component used to retrieve the schema for the current stream.",
|
1060
|
+
title="Schema Loader",
|
1077
1061
|
)
|
1078
|
-
transformations: Optional[
|
1079
|
-
List[Union[AddFields, CustomTransformation, RemoveFields]]
|
1080
|
-
] = Field(
|
1062
|
+
transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields]]] = Field(
|
1081
1063
|
None,
|
1082
|
-
description=
|
1083
|
-
title=
|
1064
|
+
description="A list of transformations to be applied to each output record.",
|
1065
|
+
title="Transformations",
|
1084
1066
|
)
|
1085
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1067
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1086
1068
|
|
1087
1069
|
|
1088
1070
|
class SessionTokenAuthenticator(BaseModel):
|
1089
|
-
type: Literal[
|
1071
|
+
type: Literal["SessionTokenAuthenticator"]
|
1090
1072
|
login_requester: HttpRequester = Field(
|
1091
1073
|
...,
|
1092
|
-
description=
|
1074
|
+
description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.",
|
1093
1075
|
examples=[
|
1094
1076
|
{
|
1095
|
-
|
1096
|
-
|
1097
|
-
|
1098
|
-
|
1099
|
-
|
1100
|
-
|
1101
|
-
|
1077
|
+
"type": "HttpRequester",
|
1078
|
+
"url_base": "https://my_api.com",
|
1079
|
+
"path": "/login",
|
1080
|
+
"authenticator": {
|
1081
|
+
"type": "BasicHttpAuthenticator",
|
1082
|
+
"username": "{{ config.username }}",
|
1083
|
+
"password": "{{ config.password }}",
|
1102
1084
|
},
|
1103
1085
|
}
|
1104
1086
|
],
|
1105
|
-
title=
|
1087
|
+
title="Login Requester",
|
1106
1088
|
)
|
1107
1089
|
session_token_path: List[str] = Field(
|
1108
1090
|
...,
|
1109
|
-
description=
|
1110
|
-
examples=[[
|
1111
|
-
title=
|
1091
|
+
description="The path in the response body returned from the login requester to the session token.",
|
1092
|
+
examples=[["access_token"], ["result", "token"]],
|
1093
|
+
title="Session Token Path",
|
1112
1094
|
)
|
1113
1095
|
expiration_duration: Optional[str] = Field(
|
1114
1096
|
None,
|
1115
|
-
description=
|
1116
|
-
examples=[
|
1117
|
-
title=
|
1097
|
+
description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.",
|
1098
|
+
examples=["PT1H", "P1D"],
|
1099
|
+
title="Expiration Duration",
|
1118
1100
|
)
|
1119
|
-
request_authentication: Union[
|
1120
|
-
SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator
|
1121
|
-
] = Field(
|
1101
|
+
request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator] = Field(
|
1122
1102
|
...,
|
1123
|
-
description=
|
1124
|
-
title=
|
1103
|
+
description="Authentication method to use for requests sent to the API, specifying how to inject the session token.",
|
1104
|
+
title="Data Request Authentication",
|
1125
1105
|
)
|
1126
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1106
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1127
1107
|
|
1128
1108
|
|
1129
1109
|
class HttpRequester(BaseModel):
|
1130
|
-
type: Literal[
|
1110
|
+
type: Literal["HttpRequester"]
|
1131
1111
|
url_base: str = Field(
|
1132
1112
|
...,
|
1133
|
-
description=
|
1113
|
+
description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
|
1134
1114
|
examples=[
|
1135
|
-
|
1115
|
+
"https://connect.squareup.com/v2",
|
1136
1116
|
"{{ config['base_url'] or 'https://app.posthog.com'}}/api/",
|
1137
1117
|
],
|
1138
|
-
title=
|
1118
|
+
title="API Base URL",
|
1139
1119
|
)
|
1140
1120
|
path: str = Field(
|
1141
1121
|
...,
|
1142
|
-
description=
|
1122
|
+
description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
|
1143
1123
|
examples=[
|
1144
|
-
|
1124
|
+
"/products",
|
1145
1125
|
"/quotes/{{ stream_partition['id'] }}/quote_line_groups",
|
1146
1126
|
"/trades/{{ config['symbol_id'] }}/history",
|
1147
1127
|
],
|
1148
|
-
title=
|
1128
|
+
title="URL Path",
|
1149
1129
|
)
|
1150
1130
|
authenticator: Optional[
|
1151
1131
|
Union[
|
@@ -1160,96 +1140,92 @@ class HttpRequester(BaseModel):
|
|
1160
1140
|
]
|
1161
1141
|
] = Field(
|
1162
1142
|
None,
|
1163
|
-
description=
|
1164
|
-
title=
|
1143
|
+
description="Authentication method to use for requests sent to the API.",
|
1144
|
+
title="Authenticator",
|
1165
1145
|
)
|
1166
|
-
error_handler: Optional[
|
1167
|
-
Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]
|
1168
|
-
] = Field(
|
1146
|
+
error_handler: Optional[Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]] = Field(
|
1169
1147
|
None,
|
1170
|
-
description=
|
1171
|
-
title=
|
1148
|
+
description="Error handler component that defines how to handle errors.",
|
1149
|
+
title="Error Handler",
|
1172
1150
|
)
|
1173
1151
|
http_method: Optional[Union[str, HttpMethodEnum]] = Field(
|
1174
|
-
|
1175
|
-
description=
|
1176
|
-
examples=[
|
1177
|
-
title=
|
1152
|
+
"GET",
|
1153
|
+
description="The HTTP method used to fetch data from the source (can be GET or POST).",
|
1154
|
+
examples=["GET", "POST"],
|
1155
|
+
title="HTTP Method",
|
1178
1156
|
)
|
1179
1157
|
request_body_data: Optional[Union[str, Dict[str, str]]] = Field(
|
1180
1158
|
None,
|
1181
|
-
description=
|
1159
|
+
description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.",
|
1182
1160
|
examples=[
|
1183
1161
|
'[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n'
|
1184
1162
|
],
|
1185
|
-
title=
|
1163
|
+
title="Request Body Payload (Non-JSON)",
|
1186
1164
|
)
|
1187
1165
|
request_body_json: Optional[Union[str, Dict[str, Any]]] = Field(
|
1188
1166
|
None,
|
1189
|
-
description=
|
1167
|
+
description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.",
|
1190
1168
|
examples=[
|
1191
|
-
{
|
1192
|
-
{
|
1193
|
-
{
|
1169
|
+
{"sort_order": "ASC", "sort_field": "CREATED_AT"},
|
1170
|
+
{"key": "{{ config['value'] }}"},
|
1171
|
+
{"sort": {"field": "updated_at", "order": "ascending"}},
|
1194
1172
|
],
|
1195
|
-
title=
|
1173
|
+
title="Request Body JSON Payload",
|
1196
1174
|
)
|
1197
1175
|
request_headers: Optional[Union[str, Dict[str, str]]] = Field(
|
1198
1176
|
None,
|
1199
|
-
description=
|
1200
|
-
examples=[{
|
1201
|
-
title=
|
1177
|
+
description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.",
|
1178
|
+
examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
|
1179
|
+
title="Request Headers",
|
1202
1180
|
)
|
1203
1181
|
request_parameters: Optional[Union[str, Dict[str, str]]] = Field(
|
1204
1182
|
None,
|
1205
|
-
description=
|
1183
|
+
description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
|
1206
1184
|
examples=[
|
1207
|
-
{
|
1185
|
+
{"unit": "day"},
|
1208
1186
|
{
|
1209
|
-
|
1187
|
+
"query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
|
1210
1188
|
},
|
1211
|
-
{
|
1212
|
-
{
|
1189
|
+
{"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
|
1190
|
+
{"sort_by[asc]": "updated_at"},
|
1213
1191
|
],
|
1214
|
-
title=
|
1192
|
+
title="Query Parameters",
|
1215
1193
|
)
|
1216
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1194
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1217
1195
|
|
1218
1196
|
|
1219
1197
|
class ParentStreamConfig(BaseModel):
|
1220
|
-
type: Literal[
|
1198
|
+
type: Literal["ParentStreamConfig"]
|
1221
1199
|
parent_key: str = Field(
|
1222
1200
|
...,
|
1223
|
-
description=
|
1224
|
-
examples=[
|
1225
|
-
title=
|
1226
|
-
)
|
1227
|
-
stream: DeclarativeStream = Field(
|
1228
|
-
..., description='Reference to the parent stream.', title='Parent Stream'
|
1201
|
+
description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.",
|
1202
|
+
examples=["id", "{{ config['parent_record_id'] }}"],
|
1203
|
+
title="Parent Key",
|
1229
1204
|
)
|
1205
|
+
stream: DeclarativeStream = Field(..., description="Reference to the parent stream.", title="Parent Stream")
|
1230
1206
|
partition_field: str = Field(
|
1231
1207
|
...,
|
1232
|
-
description=
|
1233
|
-
examples=[
|
1234
|
-
title=
|
1208
|
+
description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.",
|
1209
|
+
examples=["parent_id", "{{ config['parent_partition_field'] }}"],
|
1210
|
+
title="Current Parent Key Value Identifier",
|
1235
1211
|
)
|
1236
1212
|
request_option: Optional[RequestOption] = Field(
|
1237
1213
|
None,
|
1238
|
-
description=
|
1239
|
-
title=
|
1214
|
+
description="A request option describing where the parent key value should be injected into and under what field name if applicable.",
|
1215
|
+
title="Request Option",
|
1240
1216
|
)
|
1241
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1217
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1242
1218
|
|
1243
1219
|
|
1244
1220
|
class SimpleRetriever(BaseModel):
|
1245
|
-
type: Literal[
|
1221
|
+
type: Literal["SimpleRetriever"]
|
1246
1222
|
record_selector: RecordSelector = Field(
|
1247
1223
|
...,
|
1248
|
-
description=
|
1224
|
+
description="Component that describes how to extract records from a HTTP response.",
|
1249
1225
|
)
|
1250
1226
|
requester: Union[CustomRequester, HttpRequester] = Field(
|
1251
1227
|
...,
|
1252
|
-
description=
|
1228
|
+
description="Requester component that describes how to prepare HTTP requests to send to the source API.",
|
1253
1229
|
)
|
1254
1230
|
paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field(
|
1255
1231
|
None,
|
@@ -1260,28 +1236,24 @@ class SimpleRetriever(BaseModel):
|
|
1260
1236
|
CustomPartitionRouter,
|
1261
1237
|
ListPartitionRouter,
|
1262
1238
|
SubstreamPartitionRouter,
|
1263
|
-
List[
|
1264
|
-
Union[
|
1265
|
-
CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter
|
1266
|
-
]
|
1267
|
-
],
|
1239
|
+
List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]],
|
1268
1240
|
]
|
1269
1241
|
] = Field(
|
1270
1242
|
[],
|
1271
|
-
description=
|
1272
|
-
title=
|
1243
|
+
description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.",
|
1244
|
+
title="Partition Router",
|
1273
1245
|
)
|
1274
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1246
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1275
1247
|
|
1276
1248
|
|
1277
1249
|
class SubstreamPartitionRouter(BaseModel):
|
1278
|
-
type: Literal[
|
1250
|
+
type: Literal["SubstreamPartitionRouter"]
|
1279
1251
|
parent_stream_configs: List[ParentStreamConfig] = Field(
|
1280
1252
|
...,
|
1281
|
-
description=
|
1282
|
-
title=
|
1253
|
+
description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.",
|
1254
|
+
title="Parent Stream Configs",
|
1283
1255
|
)
|
1284
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias=
|
1256
|
+
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1285
1257
|
|
1286
1258
|
|
1287
1259
|
CompositeErrorHandler.update_forward_refs()
|