airbyte-cdk 6.48.16.post14.dev15122056170__py3-none-any.whl → 6.49.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +273 -0
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +186 -18
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +152 -4
- airbyte_cdk/sources/declarative/spec/__init__.py +2 -2
- airbyte_cdk/sources/declarative/spec/spec.py +71 -2
- airbyte_cdk/sources/declarative/transformations/config_transformations/add_fields.py +10 -23
- airbyte_cdk/sources/declarative/transformations/config_transformations/remap_field.py +2 -3
- airbyte_cdk/sources/declarative/validators/dpath_validator.py +1 -1
- {airbyte_cdk-6.48.16.post14.dev15122056170.dist-info → airbyte_cdk-6.49.0.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.48.16.post14.dev15122056170.dist-info → airbyte_cdk-6.49.0.dist-info}/RECORD +14 -14
- {airbyte_cdk-6.48.16.post14.dev15122056170.dist-info → airbyte_cdk-6.49.0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.48.16.post14.dev15122056170.dist-info → airbyte_cdk-6.49.0.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.48.16.post14.dev15122056170.dist-info → airbyte_cdk-6.49.0.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.48.16.post14.dev15122056170.dist-info → airbyte_cdk-6.49.0.dist-info}/entry_points.txt +0 -0
@@ -3803,6 +3803,61 @@ definitions:
|
|
3803
3803
|
title: Advanced Auth
|
3804
3804
|
description: Advanced specification for configuring the authentication flow.
|
3805
3805
|
"$ref": "#/definitions/AuthFlow"
|
3806
|
+
config_normalization_rules:
|
3807
|
+
title: Config Normalization Rules
|
3808
|
+
type: object
|
3809
|
+
additionalProperties: false
|
3810
|
+
properties:
|
3811
|
+
config_migrations:
|
3812
|
+
title: Config Migrations
|
3813
|
+
description: The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.
|
3814
|
+
type: array
|
3815
|
+
items:
|
3816
|
+
"$ref": "#/definitions/ConfigMigration"
|
3817
|
+
default: []
|
3818
|
+
transformations:
|
3819
|
+
title: Transformations
|
3820
|
+
description: The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.
|
3821
|
+
type: array
|
3822
|
+
items:
|
3823
|
+
anyOf:
|
3824
|
+
- "$ref": "#/definitions/ConfigRemapField"
|
3825
|
+
- "$ref": "#/definitions/ConfigAddFields"
|
3826
|
+
- "$ref": "#/definitions/ConfigRemoveFields"
|
3827
|
+
default: []
|
3828
|
+
validations:
|
3829
|
+
title: Validations
|
3830
|
+
description: The list of validations that will be performed on the incoming config at the start of each sync.
|
3831
|
+
type: array
|
3832
|
+
items:
|
3833
|
+
anyOf:
|
3834
|
+
- "$ref": "#/definitions/DpathValidator"
|
3835
|
+
- "$ref": "#/definitions/PredicateValidator"
|
3836
|
+
default: []
|
3837
|
+
ConfigMigration:
|
3838
|
+
title: Config Migration
|
3839
|
+
description: A config migration that will be applied on the incoming config at the start of a sync.
|
3840
|
+
type: object
|
3841
|
+
required:
|
3842
|
+
- type
|
3843
|
+
- transformations
|
3844
|
+
properties:
|
3845
|
+
type:
|
3846
|
+
type: string
|
3847
|
+
enum: [ConfigMigration]
|
3848
|
+
description:
|
3849
|
+
type: string
|
3850
|
+
description: The description/purpose of the config migration.
|
3851
|
+
transformations:
|
3852
|
+
title: Transformations
|
3853
|
+
description: The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.
|
3854
|
+
type: array
|
3855
|
+
items:
|
3856
|
+
anyOf:
|
3857
|
+
- "$ref": "#/definitions/ConfigRemapField"
|
3858
|
+
- "$ref": "#/definitions/ConfigAddFields"
|
3859
|
+
- "$ref": "#/definitions/ConfigRemoveFields"
|
3860
|
+
default: []
|
3806
3861
|
SubstreamPartitionRouter:
|
3807
3862
|
title: Substream Partition Router
|
3808
3863
|
description: Partition router that is used to retrieve records that have been partitioned according to records from the specified parent streams. An example of a parent stream is automobile brands and the substream would be the various car models associated with each branch.
|
@@ -4164,6 +4219,224 @@ definitions:
|
|
4164
4219
|
description: The GraphQL query to be executed
|
4165
4220
|
default: {}
|
4166
4221
|
additionalProperties: true
|
4222
|
+
DpathValidator:
|
4223
|
+
title: Dpath Validator
|
4224
|
+
description: Validator that extracts the value located at a given field path.
|
4225
|
+
type: object
|
4226
|
+
required:
|
4227
|
+
- type
|
4228
|
+
- field_path
|
4229
|
+
- validation_strategy
|
4230
|
+
properties:
|
4231
|
+
type:
|
4232
|
+
type: string
|
4233
|
+
enum: [DpathValidator]
|
4234
|
+
field_path:
|
4235
|
+
title: Field Path
|
4236
|
+
description: List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.
|
4237
|
+
type: array
|
4238
|
+
items:
|
4239
|
+
type: string
|
4240
|
+
interpolation_context:
|
4241
|
+
- config
|
4242
|
+
examples:
|
4243
|
+
- ["data"]
|
4244
|
+
- ["data", "records"]
|
4245
|
+
- ["data", "{{ parameters.name }}"]
|
4246
|
+
- ["data", "*", "record"]
|
4247
|
+
validation_strategy:
|
4248
|
+
title: Validation Strategy
|
4249
|
+
description: The condition that the specified config value will be evaluated against
|
4250
|
+
anyOf:
|
4251
|
+
- "$ref": "#/definitions/ValidateAdheresToSchema"
|
4252
|
+
PredicateValidator:
|
4253
|
+
title: Predicate Validator
|
4254
|
+
description: Validator that applies a validation strategy to a specified value.
|
4255
|
+
type: object
|
4256
|
+
required:
|
4257
|
+
- type
|
4258
|
+
- value
|
4259
|
+
- validation_strategy
|
4260
|
+
properties:
|
4261
|
+
type:
|
4262
|
+
type: string
|
4263
|
+
enum: [PredicateValidator]
|
4264
|
+
value:
|
4265
|
+
title: Value
|
4266
|
+
description: The value to be validated. Can be a literal value or interpolated from configuration.
|
4267
|
+
type:
|
4268
|
+
- string
|
4269
|
+
- number
|
4270
|
+
- object
|
4271
|
+
- array
|
4272
|
+
- boolean
|
4273
|
+
- "null"
|
4274
|
+
interpolation_context:
|
4275
|
+
- config
|
4276
|
+
examples:
|
4277
|
+
- "test-value"
|
4278
|
+
- "{{ config['api_version'] }}"
|
4279
|
+
- "{{ config['tenant_id'] }}"
|
4280
|
+
- 123
|
4281
|
+
validation_strategy:
|
4282
|
+
title: Validation Strategy
|
4283
|
+
description: The validation strategy to apply to the value.
|
4284
|
+
anyOf:
|
4285
|
+
- "$ref": "#/definitions/ValidateAdheresToSchema"
|
4286
|
+
ValidateAdheresToSchema:
|
4287
|
+
title: Validate Adheres To Schema
|
4288
|
+
description: Validates that a user-provided schema adheres to a specified JSON schema.
|
4289
|
+
type: object
|
4290
|
+
required:
|
4291
|
+
- type
|
4292
|
+
- base_schema
|
4293
|
+
properties:
|
4294
|
+
type:
|
4295
|
+
type: string
|
4296
|
+
enum: [ValidateAdheresToSchema]
|
4297
|
+
base_schema:
|
4298
|
+
title: Base JSON Schema
|
4299
|
+
description: The base JSON schema against which the user-provided schema will be validated.
|
4300
|
+
type:
|
4301
|
+
- string
|
4302
|
+
- object
|
4303
|
+
interpolation_context:
|
4304
|
+
- config
|
4305
|
+
examples:
|
4306
|
+
- "{{ config['report_validation_schema'] }}"
|
4307
|
+
- |
|
4308
|
+
'{
|
4309
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
4310
|
+
"title": "Person",
|
4311
|
+
"type": "object",
|
4312
|
+
"properties": {
|
4313
|
+
"name": {
|
4314
|
+
"type": "string",
|
4315
|
+
"description": "The person's name"
|
4316
|
+
},
|
4317
|
+
"age": {
|
4318
|
+
"type": "integer",
|
4319
|
+
"minimum": 0,
|
4320
|
+
"description": "The person's age"
|
4321
|
+
}
|
4322
|
+
},
|
4323
|
+
"required": ["name", "age"]
|
4324
|
+
}'
|
4325
|
+
- $schema: "http://json-schema.org/draft-07/schema#"
|
4326
|
+
title: Person
|
4327
|
+
type: object
|
4328
|
+
properties:
|
4329
|
+
name:
|
4330
|
+
type: string
|
4331
|
+
description: "The person's name"
|
4332
|
+
age:
|
4333
|
+
type: integer
|
4334
|
+
minimum: 0
|
4335
|
+
description: "The person's age"
|
4336
|
+
required:
|
4337
|
+
- name
|
4338
|
+
- age
|
4339
|
+
ConfigRemapField:
|
4340
|
+
title: Remap Field
|
4341
|
+
description: Transformation that remaps a field's value to another value based on a static map.
|
4342
|
+
type: object
|
4343
|
+
required:
|
4344
|
+
- type
|
4345
|
+
- map
|
4346
|
+
- field_path
|
4347
|
+
properties:
|
4348
|
+
type:
|
4349
|
+
type: string
|
4350
|
+
enum: [ConfigRemapField]
|
4351
|
+
map:
|
4352
|
+
title: Value Mapping
|
4353
|
+
description: A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.
|
4354
|
+
interpolation_context:
|
4355
|
+
- config
|
4356
|
+
type:
|
4357
|
+
- object
|
4358
|
+
- string
|
4359
|
+
additionalProperties: true
|
4360
|
+
examples:
|
4361
|
+
- pending: "in_progress"
|
4362
|
+
done: "completed"
|
4363
|
+
cancelled: "terminated"
|
4364
|
+
- "{{ config['status_mapping'] }}"
|
4365
|
+
field_path:
|
4366
|
+
title: Field Path
|
4367
|
+
description: The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.
|
4368
|
+
interpolation_context:
|
4369
|
+
- config
|
4370
|
+
type: array
|
4371
|
+
items:
|
4372
|
+
type: string
|
4373
|
+
examples:
|
4374
|
+
- ["status"]
|
4375
|
+
- ["data", "status"]
|
4376
|
+
- ["data", "{{ config.name }}", "status"]
|
4377
|
+
- ["data", "*", "status"]
|
4378
|
+
ConfigAddFields:
|
4379
|
+
title: Config Add Fields
|
4380
|
+
description: Transformation that adds fields to a config. The path of the added field can be nested.
|
4381
|
+
type: object
|
4382
|
+
required:
|
4383
|
+
- type
|
4384
|
+
- fields
|
4385
|
+
properties:
|
4386
|
+
type:
|
4387
|
+
type: string
|
4388
|
+
enum: [ConfigAddFields]
|
4389
|
+
fields:
|
4390
|
+
title: Fields
|
4391
|
+
description: A list of transformations (path and corresponding value) that will be added to the config.
|
4392
|
+
type: array
|
4393
|
+
items:
|
4394
|
+
"$ref": "#/definitions/AddedFieldDefinition"
|
4395
|
+
condition:
|
4396
|
+
description: Fields will be added if expression is evaluated to True.
|
4397
|
+
type: string
|
4398
|
+
default: ""
|
4399
|
+
interpolation_context:
|
4400
|
+
- config
|
4401
|
+
- property
|
4402
|
+
examples:
|
4403
|
+
- "{{ config['environemnt'] == 'sandbox' }}"
|
4404
|
+
- "{{ property is integer }}"
|
4405
|
+
- "{{ property|length > 5 }}"
|
4406
|
+
- "{{ property == 'some_string_to_match' }}"
|
4407
|
+
ConfigRemoveFields:
|
4408
|
+
title: Config Remove Fields
|
4409
|
+
description: Transformation that removes a field from the config.
|
4410
|
+
type: object
|
4411
|
+
required:
|
4412
|
+
- type
|
4413
|
+
- field_pointers
|
4414
|
+
properties:
|
4415
|
+
type:
|
4416
|
+
type: string
|
4417
|
+
enum: [ConfigRemoveFields]
|
4418
|
+
field_pointers:
|
4419
|
+
title: Field Pointers
|
4420
|
+
description: A list of field pointers to be removed from the config.
|
4421
|
+
type: array
|
4422
|
+
items:
|
4423
|
+
items:
|
4424
|
+
type: string
|
4425
|
+
examples:
|
4426
|
+
- ["tags"]
|
4427
|
+
- [["content", "html"], ["content", "plain_text"]]
|
4428
|
+
condition:
|
4429
|
+
description: Fields will be removed if expression is evaluated to True.
|
4430
|
+
type: string
|
4431
|
+
default: ""
|
4432
|
+
interpolation_context:
|
4433
|
+
- config
|
4434
|
+
- property
|
4435
|
+
examples:
|
4436
|
+
- "{{ config['environemnt'] == 'sandbox' }}"
|
4437
|
+
- "{{ property is integer }}"
|
4438
|
+
- "{{ property|length > 5 }}"
|
4439
|
+
- "{{ property == 'some_string_to_match' }}"
|
4167
4440
|
interpolation:
|
4168
4441
|
variables:
|
4169
4442
|
- title: config
|
@@ -1523,6 +1523,77 @@ class RequestBodyGraphQlQuery(BaseModel):
|
|
1523
1523
|
query: Dict[str, Any] = Field(..., description="The GraphQL query to be executed")
|
1524
1524
|
|
1525
1525
|
|
1526
|
+
class ValidateAdheresToSchema(BaseModel):
|
1527
|
+
type: Literal["ValidateAdheresToSchema"]
|
1528
|
+
base_schema: Union[str, Dict[str, Any]] = Field(
|
1529
|
+
...,
|
1530
|
+
description="The base JSON schema against which the user-provided schema will be validated.",
|
1531
|
+
examples=[
|
1532
|
+
"{{ config['report_validation_schema'] }}",
|
1533
|
+
'\'{\n "$schema": "http://json-schema.org/draft-07/schema#",\n "title": "Person",\n "type": "object",\n "properties": {\n "name": {\n "type": "string",\n "description": "The person\'s name"\n },\n "age": {\n "type": "integer",\n "minimum": 0,\n "description": "The person\'s age"\n }\n },\n "required": ["name", "age"]\n}\'\n',
|
1534
|
+
{
|
1535
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
1536
|
+
"title": "Person",
|
1537
|
+
"type": "object",
|
1538
|
+
"properties": {
|
1539
|
+
"name": {"type": "string", "description": "The person's name"},
|
1540
|
+
"age": {
|
1541
|
+
"type": "integer",
|
1542
|
+
"minimum": 0,
|
1543
|
+
"description": "The person's age",
|
1544
|
+
},
|
1545
|
+
},
|
1546
|
+
"required": ["name", "age"],
|
1547
|
+
},
|
1548
|
+
],
|
1549
|
+
title="Base JSON Schema",
|
1550
|
+
)
|
1551
|
+
|
1552
|
+
|
1553
|
+
class ConfigRemapField(BaseModel):
|
1554
|
+
type: Literal["ConfigRemapField"]
|
1555
|
+
map: Union[Dict[str, Any], str] = Field(
|
1556
|
+
...,
|
1557
|
+
description="A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.",
|
1558
|
+
examples=[
|
1559
|
+
{"pending": "in_progress", "done": "completed", "cancelled": "terminated"},
|
1560
|
+
"{{ config['status_mapping'] }}",
|
1561
|
+
],
|
1562
|
+
title="Value Mapping",
|
1563
|
+
)
|
1564
|
+
field_path: List[str] = Field(
|
1565
|
+
...,
|
1566
|
+
description="The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.",
|
1567
|
+
examples=[
|
1568
|
+
["status"],
|
1569
|
+
["data", "status"],
|
1570
|
+
["data", "{{ config.name }}", "status"],
|
1571
|
+
["data", "*", "status"],
|
1572
|
+
],
|
1573
|
+
title="Field Path",
|
1574
|
+
)
|
1575
|
+
|
1576
|
+
|
1577
|
+
class ConfigRemoveFields(BaseModel):
|
1578
|
+
type: Literal["ConfigRemoveFields"]
|
1579
|
+
field_pointers: List[List[str]] = Field(
|
1580
|
+
...,
|
1581
|
+
description="A list of field pointers to be removed from the config.",
|
1582
|
+
examples=[["tags"], [["content", "html"], ["content", "plain_text"]]],
|
1583
|
+
title="Field Pointers",
|
1584
|
+
)
|
1585
|
+
condition: Optional[str] = Field(
|
1586
|
+
"",
|
1587
|
+
description="Fields will be removed if expression is evaluated to True.",
|
1588
|
+
examples=[
|
1589
|
+
"{{ config['environemnt'] == 'sandbox' }}",
|
1590
|
+
"{{ property is integer }}",
|
1591
|
+
"{{ property|length > 5 }}",
|
1592
|
+
"{{ property == 'some_string_to_match' }}",
|
1593
|
+
],
|
1594
|
+
)
|
1595
|
+
|
1596
|
+
|
1526
1597
|
class AddedFieldDefinition(BaseModel):
|
1527
1598
|
type: Literal["AddedFieldDefinition"]
|
1528
1599
|
path: List[str] = Field(
|
@@ -1905,29 +1976,68 @@ class GzipDecoder(BaseModel):
|
|
1905
1976
|
decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonlDecoder]
|
1906
1977
|
|
1907
1978
|
|
1908
|
-
class
|
1909
|
-
type: Literal["
|
1910
|
-
|
1979
|
+
class RequestBodyGraphQL(BaseModel):
|
1980
|
+
type: Literal["RequestBodyGraphQL"]
|
1981
|
+
value: RequestBodyGraphQlQuery
|
1982
|
+
|
1983
|
+
|
1984
|
+
class DpathValidator(BaseModel):
|
1985
|
+
type: Literal["DpathValidator"]
|
1986
|
+
field_path: List[str] = Field(
|
1911
1987
|
...,
|
1912
|
-
description=
|
1913
|
-
|
1988
|
+
description='List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.',
|
1989
|
+
examples=[
|
1990
|
+
["data"],
|
1991
|
+
["data", "records"],
|
1992
|
+
["data", "{{ parameters.name }}"],
|
1993
|
+
["data", "*", "record"],
|
1994
|
+
],
|
1995
|
+
title="Field Path",
|
1914
1996
|
)
|
1915
|
-
|
1916
|
-
|
1917
|
-
description="
|
1918
|
-
|
1919
|
-
title="Documentation URL",
|
1997
|
+
validation_strategy: ValidateAdheresToSchema = Field(
|
1998
|
+
...,
|
1999
|
+
description="The condition that the specified config value will be evaluated against",
|
2000
|
+
title="Validation Strategy",
|
1920
2001
|
)
|
1921
|
-
|
1922
|
-
|
1923
|
-
|
1924
|
-
|
2002
|
+
|
2003
|
+
|
2004
|
+
class PredicateValidator(BaseModel):
|
2005
|
+
type: Literal["PredicateValidator"]
|
2006
|
+
value: Optional[Union[str, float, Dict[str, Any], List[Any], bool]] = Field(
|
2007
|
+
...,
|
2008
|
+
description="The value to be validated. Can be a literal value or interpolated from configuration.",
|
2009
|
+
examples=[
|
2010
|
+
"test-value",
|
2011
|
+
"{{ config['api_version'] }}",
|
2012
|
+
"{{ config['tenant_id'] }}",
|
2013
|
+
123,
|
2014
|
+
],
|
2015
|
+
title="Value",
|
2016
|
+
)
|
2017
|
+
validation_strategy: ValidateAdheresToSchema = Field(
|
2018
|
+
...,
|
2019
|
+
description="The validation strategy to apply to the value.",
|
2020
|
+
title="Validation Strategy",
|
1925
2021
|
)
|
1926
2022
|
|
1927
2023
|
|
1928
|
-
class
|
1929
|
-
type: Literal["
|
1930
|
-
|
2024
|
+
class ConfigAddFields(BaseModel):
|
2025
|
+
type: Literal["ConfigAddFields"]
|
2026
|
+
fields: List[AddedFieldDefinition] = Field(
|
2027
|
+
...,
|
2028
|
+
description="A list of transformations (path and corresponding value) that will be added to the config.",
|
2029
|
+
title="Fields",
|
2030
|
+
)
|
2031
|
+
condition: Optional[str] = Field(
|
2032
|
+
"",
|
2033
|
+
description="Fields will be added if expression is evaluated to True.",
|
2034
|
+
examples=[
|
2035
|
+
"{{ config['environemnt'] == 'sandbox' }}",
|
2036
|
+
"{{ property is integer }}",
|
2037
|
+
"{{ property|length > 5 }}",
|
2038
|
+
"{{ property == 'some_string_to_match' }}",
|
2039
|
+
],
|
2040
|
+
)
|
1931
2041
|
|
1932
2042
|
|
1933
2043
|
class CompositeErrorHandler(BaseModel):
|
@@ -1985,6 +2095,64 @@ class ZipfileDecoder(BaseModel):
|
|
1985
2095
|
)
|
1986
2096
|
|
1987
2097
|
|
2098
|
+
class ConfigMigration(BaseModel):
|
2099
|
+
type: Literal["ConfigMigration"]
|
2100
|
+
description: Optional[str] = Field(
|
2101
|
+
None, description="The description/purpose of the config migration."
|
2102
|
+
)
|
2103
|
+
transformations: List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]] = Field(
|
2104
|
+
...,
|
2105
|
+
description="The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.",
|
2106
|
+
title="Transformations",
|
2107
|
+
)
|
2108
|
+
|
2109
|
+
|
2110
|
+
class ConfigNormalizationRules(BaseModel):
|
2111
|
+
class Config:
|
2112
|
+
extra = Extra.forbid
|
2113
|
+
|
2114
|
+
config_migrations: Optional[List[ConfigMigration]] = Field(
|
2115
|
+
[],
|
2116
|
+
description="The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.",
|
2117
|
+
title="Config Migrations",
|
2118
|
+
)
|
2119
|
+
transformations: Optional[
|
2120
|
+
List[Union[ConfigRemapField, ConfigAddFields, ConfigRemoveFields]]
|
2121
|
+
] = Field(
|
2122
|
+
[],
|
2123
|
+
description="The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.",
|
2124
|
+
title="Transformations",
|
2125
|
+
)
|
2126
|
+
validations: Optional[List[Union[DpathValidator, PredicateValidator]]] = Field(
|
2127
|
+
[],
|
2128
|
+
description="The list of validations that will be performed on the incoming config at the start of each sync.",
|
2129
|
+
title="Validations",
|
2130
|
+
)
|
2131
|
+
|
2132
|
+
|
2133
|
+
class Spec(BaseModel):
|
2134
|
+
type: Literal["Spec"]
|
2135
|
+
connection_specification: Dict[str, Any] = Field(
|
2136
|
+
...,
|
2137
|
+
description="A connection specification describing how a the connector can be configured.",
|
2138
|
+
title="Connection Specification",
|
2139
|
+
)
|
2140
|
+
documentation_url: Optional[str] = Field(
|
2141
|
+
None,
|
2142
|
+
description="URL of the connector's documentation page.",
|
2143
|
+
examples=["https://docs.airbyte.com/integrations/sources/dremio"],
|
2144
|
+
title="Documentation URL",
|
2145
|
+
)
|
2146
|
+
advanced_auth: Optional[AuthFlow] = Field(
|
2147
|
+
None,
|
2148
|
+
description="Advanced specification for configuring the authentication flow.",
|
2149
|
+
title="Advanced Auth",
|
2150
|
+
)
|
2151
|
+
config_normalization_rules: Optional[ConfigNormalizationRules] = Field(
|
2152
|
+
None, title="Config Normalization Rules"
|
2153
|
+
)
|
2154
|
+
|
2155
|
+
|
1988
2156
|
class DeclarativeSource1(BaseModel):
|
1989
2157
|
class Config:
|
1990
2158
|
extra = Extra.forbid
|
@@ -2164,7 +2332,7 @@ class DeclarativeStream(BaseModel):
|
|
2164
2332
|
]
|
2165
2333
|
] = Field(
|
2166
2334
|
None,
|
2167
|
-
description="
|
2335
|
+
description="One or many schema loaders can be used to retrieve the schema for the current stream. When multiple schema loaders are defined, schema properties will be merged together. Schema loaders defined first taking precedence in the event of a conflict.",
|
2168
2336
|
title="Schema Loader",
|
2169
2337
|
)
|
2170
2338
|
transformations: Optional[
|
@@ -19,6 +19,7 @@ from typing import (
|
|
19
19
|
Optional,
|
20
20
|
Type,
|
21
21
|
Union,
|
22
|
+
cast,
|
22
23
|
get_args,
|
23
24
|
get_origin,
|
24
25
|
get_type_hints,
|
@@ -33,6 +34,7 @@ from airbyte_cdk.connector_builder.models import (
|
|
33
34
|
)
|
34
35
|
from airbyte_cdk.models import FailureType, Level
|
35
36
|
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
37
|
+
from airbyte_cdk.sources.declarative import transformations
|
36
38
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
|
37
39
|
from airbyte_cdk.sources.declarative.async_job.job_tracker import JobTracker
|
38
40
|
from airbyte_cdk.sources.declarative.async_job.repository import AsyncJobRepository
|
@@ -154,9 +156,21 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
154
156
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
155
157
|
ConcurrencyLevel as ConcurrencyLevelModel,
|
156
158
|
)
|
159
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
160
|
+
ConfigAddFields as ConfigAddFieldsModel,
|
161
|
+
)
|
157
162
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
158
163
|
ConfigComponentsResolver as ConfigComponentsResolverModel,
|
159
164
|
)
|
165
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
166
|
+
ConfigMigration as ConfigMigrationModel,
|
167
|
+
)
|
168
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
169
|
+
ConfigRemapField as ConfigRemapFieldModel,
|
170
|
+
)
|
171
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
172
|
+
ConfigRemoveFields as ConfigRemoveFieldsModel,
|
173
|
+
)
|
160
174
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
161
175
|
ConstantBackoffStrategy as ConstantBackoffStrategyModel,
|
162
176
|
)
|
@@ -226,6 +240,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
226
240
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
227
241
|
DpathFlattenFields as DpathFlattenFieldsModel,
|
228
242
|
)
|
243
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
244
|
+
DpathValidator as DpathValidatorModel,
|
245
|
+
)
|
229
246
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
230
247
|
DynamicSchemaLoader as DynamicSchemaLoaderModel,
|
231
248
|
)
|
@@ -337,6 +354,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
337
354
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
338
355
|
ParentStreamConfig as ParentStreamConfigModel,
|
339
356
|
)
|
357
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
358
|
+
PredicateValidator as PredicateValidatorModel,
|
359
|
+
)
|
340
360
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
341
361
|
PropertiesFromEndpoint as PropertiesFromEndpointModel,
|
342
362
|
)
|
@@ -401,6 +421,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
|
|
401
421
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
402
422
|
UnlimitedCallRatePolicy as UnlimitedCallRatePolicyModel,
|
403
423
|
)
|
424
|
+
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
425
|
+
ValidateAdheresToSchema as ValidateAdheresToSchemaModel,
|
426
|
+
)
|
404
427
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType
|
405
428
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
|
406
429
|
WaitTimeFromHeader as WaitTimeFromHeaderModel,
|
@@ -476,7 +499,7 @@ from airbyte_cdk.sources.declarative.requesters.request_options import (
|
|
476
499
|
RequestOptionsProvider,
|
477
500
|
)
|
478
501
|
from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
|
479
|
-
from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
|
502
|
+
from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester
|
480
503
|
from airbyte_cdk.sources.declarative.resolvers import (
|
481
504
|
ComponentMappingDefinition,
|
482
505
|
ConfigComponentsResolver,
|
@@ -506,7 +529,7 @@ from airbyte_cdk.sources.declarative.schema import (
|
|
506
529
|
TypesMap,
|
507
530
|
)
|
508
531
|
from airbyte_cdk.sources.declarative.schema.composite_schema_loader import CompositeSchemaLoader
|
509
|
-
from airbyte_cdk.sources.declarative.spec import Spec
|
532
|
+
from airbyte_cdk.sources.declarative.spec import ConfigMigration, Spec
|
510
533
|
from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer
|
511
534
|
from airbyte_cdk.sources.declarative.transformations import (
|
512
535
|
AddFields,
|
@@ -514,6 +537,14 @@ from airbyte_cdk.sources.declarative.transformations import (
|
|
514
537
|
RemoveFields,
|
515
538
|
)
|
516
539
|
from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
|
540
|
+
from airbyte_cdk.sources.declarative.transformations.config_transformations import (
|
541
|
+
ConfigAddFields,
|
542
|
+
ConfigRemapField,
|
543
|
+
ConfigRemoveFields,
|
544
|
+
)
|
545
|
+
from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
|
546
|
+
ConfigTransformation,
|
547
|
+
)
|
517
548
|
from airbyte_cdk.sources.declarative.transformations.dpath_flatten_fields import (
|
518
549
|
DpathFlattenFields,
|
519
550
|
KeyTransformation,
|
@@ -530,6 +561,11 @@ from airbyte_cdk.sources.declarative.transformations.keys_to_lower_transformatio
|
|
530
561
|
from airbyte_cdk.sources.declarative.transformations.keys_to_snake_transformation import (
|
531
562
|
KeysToSnakeCaseTransformation,
|
532
563
|
)
|
564
|
+
from airbyte_cdk.sources.declarative.validators import (
|
565
|
+
DpathValidator,
|
566
|
+
PredicateValidator,
|
567
|
+
ValidateAdheresToSchema,
|
568
|
+
)
|
533
569
|
from airbyte_cdk.sources.http_logger import format_http_message
|
534
570
|
from airbyte_cdk.sources.message import (
|
535
571
|
InMemoryMessageRepository,
|
@@ -618,6 +654,10 @@ class ModelToComponentFactory:
|
|
618
654
|
CheckDynamicStreamModel: self.create_check_dynamic_stream,
|
619
655
|
CompositeErrorHandlerModel: self.create_composite_error_handler,
|
620
656
|
ConcurrencyLevelModel: self.create_concurrency_level,
|
657
|
+
ConfigMigrationModel: self.create_config_migration,
|
658
|
+
ConfigAddFieldsModel: self.create_config_add_fields,
|
659
|
+
ConfigRemapFieldModel: self.create_config_remap_field,
|
660
|
+
ConfigRemoveFieldsModel: self.create_config_remove_fields,
|
621
661
|
ConstantBackoffStrategyModel: self.create_constant_backoff_strategy,
|
622
662
|
CsvDecoderModel: self.create_csv_decoder,
|
623
663
|
CursorPaginationModel: self.create_cursor_pagination,
|
@@ -641,6 +681,7 @@ class ModelToComponentFactory:
|
|
641
681
|
DefaultErrorHandlerModel: self.create_default_error_handler,
|
642
682
|
DefaultPaginatorModel: self.create_default_paginator,
|
643
683
|
DpathExtractorModel: self.create_dpath_extractor,
|
684
|
+
DpathValidatorModel: self.create_dpath_validator,
|
644
685
|
ResponseToFileExtractorModel: self.create_response_to_file_extractor,
|
645
686
|
ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy,
|
646
687
|
SessionTokenAuthenticatorModel: self.create_session_token_authenticator,
|
@@ -674,6 +715,7 @@ class ModelToComponentFactory:
|
|
674
715
|
OffsetIncrementModel: self.create_offset_increment,
|
675
716
|
PageIncrementModel: self.create_page_increment,
|
676
717
|
ParentStreamConfigModel: self.create_parent_stream_config,
|
718
|
+
PredicateValidatorModel: self.create_predicate_validator,
|
677
719
|
PropertiesFromEndpointModel: self.create_properties_from_endpoint,
|
678
720
|
PropertyChunkingModel: self.create_property_chunking,
|
679
721
|
QueryPropertiesModel: self.create_query_properties,
|
@@ -688,6 +730,7 @@ class ModelToComponentFactory:
|
|
688
730
|
StateDelegatingStreamModel: self.create_state_delegating_stream,
|
689
731
|
SpecModel: self.create_spec,
|
690
732
|
SubstreamPartitionRouterModel: self.create_substream_partition_router,
|
733
|
+
ValidateAdheresToSchemaModel: self.create_validate_adheres_to_schema,
|
691
734
|
WaitTimeFromHeaderModel: self.create_wait_time_from_header,
|
692
735
|
WaitUntilTimeFromHeaderModel: self.create_wait_until_time_from_header,
|
693
736
|
AsyncRetrieverModel: self.create_async_retriever,
|
@@ -780,6 +823,75 @@ class ModelToComponentFactory:
|
|
780
823
|
if log not in self._collected_deprecation_logs:
|
781
824
|
self._collected_deprecation_logs.append(log)
|
782
825
|
|
826
|
+
def create_config_migration(
|
827
|
+
self, model: ConfigMigrationModel, config: Config
|
828
|
+
) -> ConfigMigration:
|
829
|
+
transformations: List[ConfigTransformation] = [
|
830
|
+
self._create_component_from_model(transformation, config)
|
831
|
+
for transformation in model.transformations
|
832
|
+
]
|
833
|
+
|
834
|
+
return ConfigMigration(
|
835
|
+
description=model.description,
|
836
|
+
transformations=transformations,
|
837
|
+
)
|
838
|
+
|
839
|
+
def create_config_add_fields(
|
840
|
+
self, model: ConfigAddFieldsModel, config: Config, **kwargs: Any
|
841
|
+
) -> ConfigAddFields:
|
842
|
+
fields = [self._create_component_from_model(field, config) for field in model.fields]
|
843
|
+
return ConfigAddFields(
|
844
|
+
fields=fields,
|
845
|
+
condition=model.condition or "",
|
846
|
+
)
|
847
|
+
|
848
|
+
@staticmethod
|
849
|
+
def create_config_remove_fields(
|
850
|
+
model: ConfigRemoveFieldsModel, config: Config, **kwargs: Any
|
851
|
+
) -> ConfigRemoveFields:
|
852
|
+
return ConfigRemoveFields(
|
853
|
+
field_pointers=model.field_pointers,
|
854
|
+
condition=model.condition or "",
|
855
|
+
)
|
856
|
+
|
857
|
+
@staticmethod
|
858
|
+
def create_config_remap_field(
|
859
|
+
model: ConfigRemapFieldModel, config: Config, **kwargs: Any
|
860
|
+
) -> ConfigRemapField:
|
861
|
+
mapping = cast(Mapping[str, Any], model.map)
|
862
|
+
return ConfigRemapField(
|
863
|
+
map=mapping,
|
864
|
+
field_path=model.field_path,
|
865
|
+
config=config,
|
866
|
+
)
|
867
|
+
|
868
|
+
def create_dpath_validator(self, model: DpathValidatorModel, config: Config) -> DpathValidator:
|
869
|
+
strategy = self._create_component_from_model(model.validation_strategy, config)
|
870
|
+
|
871
|
+
return DpathValidator(
|
872
|
+
field_path=model.field_path,
|
873
|
+
strategy=strategy,
|
874
|
+
)
|
875
|
+
|
876
|
+
def create_predicate_validator(
|
877
|
+
self, model: PredicateValidatorModel, config: Config
|
878
|
+
) -> PredicateValidator:
|
879
|
+
strategy = self._create_component_from_model(model.validation_strategy, config)
|
880
|
+
|
881
|
+
return PredicateValidator(
|
882
|
+
value=model.value,
|
883
|
+
strategy=strategy,
|
884
|
+
)
|
885
|
+
|
886
|
+
@staticmethod
|
887
|
+
def create_validate_adheres_to_schema(
|
888
|
+
model: ValidateAdheresToSchemaModel, config: Config, **kwargs: Any
|
889
|
+
) -> ValidateAdheresToSchema:
|
890
|
+
base_schema = cast(Mapping[str, Any], model.base_schema)
|
891
|
+
return ValidateAdheresToSchema(
|
892
|
+
schema=base_schema,
|
893
|
+
)
|
894
|
+
|
783
895
|
@staticmethod
|
784
896
|
def create_added_field_definition(
|
785
897
|
model: AddedFieldDefinitionModel, config: Config, **kwargs: Any
|
@@ -3495,13 +3607,49 @@ class ModelToComponentFactory:
|
|
3495
3607
|
parameters=model.parameters or {},
|
3496
3608
|
)
|
3497
3609
|
|
3498
|
-
|
3499
|
-
|
3610
|
+
def create_spec(self, model: SpecModel, config: Config, **kwargs: Any) -> Spec:
|
3611
|
+
config_migrations = [
|
3612
|
+
self._create_component_from_model(migration, config)
|
3613
|
+
for migration in (
|
3614
|
+
model.config_normalization_rules.config_migrations
|
3615
|
+
if (
|
3616
|
+
model.config_normalization_rules
|
3617
|
+
and model.config_normalization_rules.config_migrations
|
3618
|
+
)
|
3619
|
+
else []
|
3620
|
+
)
|
3621
|
+
]
|
3622
|
+
config_transformations = [
|
3623
|
+
self._create_component_from_model(transformation, config)
|
3624
|
+
for transformation in (
|
3625
|
+
model.config_normalization_rules.transformations
|
3626
|
+
if (
|
3627
|
+
model.config_normalization_rules
|
3628
|
+
and model.config_normalization_rules.transformations
|
3629
|
+
)
|
3630
|
+
else []
|
3631
|
+
)
|
3632
|
+
]
|
3633
|
+
config_validations = [
|
3634
|
+
self._create_component_from_model(validation, config)
|
3635
|
+
for validation in (
|
3636
|
+
model.config_normalization_rules.validations
|
3637
|
+
if (
|
3638
|
+
model.config_normalization_rules
|
3639
|
+
and model.config_normalization_rules.validations
|
3640
|
+
)
|
3641
|
+
else []
|
3642
|
+
)
|
3643
|
+
]
|
3644
|
+
|
3500
3645
|
return Spec(
|
3501
3646
|
connection_specification=model.connection_specification,
|
3502
3647
|
documentation_url=model.documentation_url,
|
3503
3648
|
advanced_auth=model.advanced_auth,
|
3504
3649
|
parameters={},
|
3650
|
+
config_migrations=config_migrations,
|
3651
|
+
config_transformations=config_transformations,
|
3652
|
+
config_validations=config_validations,
|
3505
3653
|
)
|
3506
3654
|
|
3507
3655
|
def create_substream_partition_router(
|
@@ -2,15 +2,33 @@
|
|
2
2
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
|
-
|
6
|
-
from
|
5
|
+
import json
|
6
|
+
from dataclasses import InitVar, dataclass, field
|
7
|
+
from typing import Any, List, Mapping, MutableMapping, Optional
|
7
8
|
|
9
|
+
import orjson
|
10
|
+
|
11
|
+
from airbyte_cdk.config_observation import create_connector_config_control_message
|
12
|
+
from airbyte_cdk.entrypoint import AirbyteEntrypoint
|
8
13
|
from airbyte_cdk.models import (
|
9
14
|
AdvancedAuth,
|
10
15
|
ConnectorSpecification,
|
11
16
|
ConnectorSpecificationSerializer,
|
12
17
|
)
|
18
|
+
from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerializer
|
13
19
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import AuthFlow
|
20
|
+
from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
|
21
|
+
ConfigTransformation,
|
22
|
+
)
|
23
|
+
from airbyte_cdk.sources.declarative.validators.validator import Validator
|
24
|
+
from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
|
25
|
+
from airbyte_cdk.sources.source import Source
|
26
|
+
|
27
|
+
|
28
|
+
@dataclass
|
29
|
+
class ConfigMigration:
|
30
|
+
transformations: List[ConfigTransformation]
|
31
|
+
description: Optional[str] = None
|
14
32
|
|
15
33
|
|
16
34
|
@dataclass
|
@@ -27,6 +45,10 @@ class Spec:
|
|
27
45
|
parameters: InitVar[Mapping[str, Any]]
|
28
46
|
documentation_url: Optional[str] = None
|
29
47
|
advanced_auth: Optional[AuthFlow] = None
|
48
|
+
config_migrations: List[ConfigMigration] = field(default_factory=list)
|
49
|
+
config_transformations: List[ConfigTransformation] = field(default_factory=list)
|
50
|
+
config_validations: List[Validator] = field(default_factory=list)
|
51
|
+
message_repository: MessageRepository = InMemoryMessageRepository()
|
30
52
|
|
31
53
|
def generate_spec(self) -> ConnectorSpecification:
|
32
54
|
"""
|
@@ -46,3 +68,50 @@ class Spec:
|
|
46
68
|
|
47
69
|
# We remap these keys to camel case because that's the existing format expected by the rest of the platform
|
48
70
|
return ConnectorSpecificationSerializer.load(obj)
|
71
|
+
|
72
|
+
def migrate_config(
|
73
|
+
self, args: List[str], source: Source, config: MutableMapping[str, Any]
|
74
|
+
) -> None:
|
75
|
+
"""
|
76
|
+
Apply all specified config transformations to the provided config and save the modified config to the given path and emit a control message.
|
77
|
+
|
78
|
+
:param args: Command line arguments
|
79
|
+
:param source: Source instance
|
80
|
+
:param config: The user-provided config to migrate
|
81
|
+
"""
|
82
|
+
config_path = AirbyteEntrypoint(source).extract_config(args)
|
83
|
+
|
84
|
+
if not config_path:
|
85
|
+
return
|
86
|
+
|
87
|
+
mutable_config = dict(config)
|
88
|
+
for migration in self.config_migrations:
|
89
|
+
for transformation in migration.transformations:
|
90
|
+
transformation.transform(mutable_config)
|
91
|
+
|
92
|
+
if mutable_config != config:
|
93
|
+
with open(config_path, "w") as f:
|
94
|
+
json.dump(mutable_config, f)
|
95
|
+
self.message_repository.emit_message(
|
96
|
+
create_connector_config_control_message(mutable_config)
|
97
|
+
)
|
98
|
+
for message in self.message_repository.consume_queue():
|
99
|
+
print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
|
100
|
+
|
101
|
+
def transform_config(self, config: MutableMapping[str, Any]) -> None:
|
102
|
+
"""
|
103
|
+
Apply all config transformations to the provided config.
|
104
|
+
|
105
|
+
:param config: The user-provided configuration
|
106
|
+
"""
|
107
|
+
for transformation in self.config_transformations:
|
108
|
+
transformation.transform(config)
|
109
|
+
|
110
|
+
def validate_config(self, config: MutableMapping[str, Any]) -> None:
|
111
|
+
"""
|
112
|
+
Apply all config validations to the provided config.
|
113
|
+
|
114
|
+
:param config: The user-provided configuration
|
115
|
+
"""
|
116
|
+
for validator in self.config_validations:
|
117
|
+
validator.validate(config)
|
@@ -9,28 +9,13 @@ import dpath
|
|
9
9
|
|
10
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
11
11
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
12
|
+
from airbyte_cdk.sources.declarative.transformations.add_fields import (
|
13
|
+
AddedFieldDefinition,
|
14
|
+
ParsedAddFieldDefinition,
|
15
|
+
)
|
12
16
|
from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
|
13
17
|
ConfigTransformation,
|
14
18
|
)
|
15
|
-
from airbyte_cdk.sources.types import FieldPointer
|
16
|
-
|
17
|
-
|
18
|
-
@dataclass(frozen=True)
|
19
|
-
class AddedFieldDefinition:
|
20
|
-
"""Defines the field to add on a config"""
|
21
|
-
|
22
|
-
path: FieldPointer
|
23
|
-
value: Union[InterpolatedString, str]
|
24
|
-
value_type: Optional[Type[Any]] = None
|
25
|
-
|
26
|
-
|
27
|
-
@dataclass(frozen=True)
|
28
|
-
class ParsedAddFieldDefinition:
|
29
|
-
"""Defines the field to add on a config"""
|
30
|
-
|
31
|
-
path: FieldPointer
|
32
|
-
value: InterpolatedString
|
33
|
-
value_type: Optional[Type[Any]] = None
|
34
19
|
|
35
20
|
|
36
21
|
@dataclass
|
@@ -44,19 +29,19 @@ class ConfigAddFields(ConfigTransformation):
|
|
44
29
|
Examples of instantiating this transformation via YAML:
|
45
30
|
- type: ConfigAddFields
|
46
31
|
fields:
|
47
|
-
|
32
|
+
### hardcoded constant
|
48
33
|
- path: ["path"]
|
49
34
|
value: "static_value"
|
50
35
|
|
51
|
-
|
36
|
+
### nested path
|
52
37
|
- path: ["path", "to", "field"]
|
53
38
|
value: "static"
|
54
39
|
|
55
|
-
|
40
|
+
### from config
|
56
41
|
- path: ["derived_field"]
|
57
42
|
value: "{{ config.original_field }}"
|
58
43
|
|
59
|
-
|
44
|
+
### by supplying any valid Jinja template directive or expression
|
60
45
|
- path: ["two_times_two"]
|
61
46
|
value: "{{ 2 * 2 }}"
|
62
47
|
|
@@ -90,6 +75,7 @@ class ConfigAddFields(ConfigTransformation):
|
|
90
75
|
add_field.path,
|
91
76
|
InterpolatedString.create(add_field.value, parameters={}),
|
92
77
|
value_type=add_field.value_type,
|
78
|
+
parameters={},
|
93
79
|
)
|
94
80
|
)
|
95
81
|
else:
|
@@ -98,6 +84,7 @@ class ConfigAddFields(ConfigTransformation):
|
|
98
84
|
add_field.path,
|
99
85
|
add_field.value,
|
100
86
|
value_type=add_field.value_type,
|
87
|
+
parameters={},
|
101
88
|
)
|
102
89
|
)
|
103
90
|
|
@@ -3,9 +3,8 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
from dataclasses import dataclass, field
|
6
|
-
from typing import Any, List, Mapping, MutableMapping
|
6
|
+
from typing import Any, List, Mapping, MutableMapping
|
7
7
|
|
8
|
-
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
9
8
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
|
10
9
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
11
10
|
from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
|
@@ -20,7 +19,7 @@ class ConfigRemapField(ConfigTransformation):
|
|
20
19
|
"""
|
21
20
|
|
22
21
|
map: Mapping[str, Any]
|
23
|
-
field_path: List[
|
22
|
+
field_path: List[str]
|
24
23
|
config: Mapping[str, Any] = field(default_factory=dict)
|
25
24
|
|
26
25
|
def __post_init__(self) -> None:
|
@@ -89,7 +89,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=GoZJ8Oxb
|
|
89
89
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
90
90
|
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
|
91
91
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
92
|
-
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=
|
92
|
+
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=ptEGR_DdB0DXZlFHxcXcW1jwBLu_wonoeJsXSw1Bgnw,176338
|
93
93
|
airbyte_cdk/sources/declarative/declarative_source.py,sha256=qmyMnnet92eGc3C22yBtpvD5UZjqdhsAafP_zxI5wp8,1814
|
94
94
|
airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
|
95
95
|
airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
|
@@ -133,14 +133,14 @@ airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migrati
|
|
133
133
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
134
134
|
airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
|
135
135
|
airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py,sha256=Imnj3yef0aqRdLfaUxkIYISUb8YkiPrRH_wBd-x8HjM,5999
|
136
|
-
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=
|
136
|
+
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=NLMi8gVD1DK_jl_9LwSu-RMaxn410XC_GlbEgKGh6js,124938
|
137
137
|
airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
138
138
|
airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
|
139
139
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9Rbu5OexXSDN9QWDo8YU4tT9M2LDVOgGA,802
|
140
140
|
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=2UdpCz3yi7ISZTyqkQXSSy3dMxeyOWqV7OlAS5b9GVg,11568
|
141
141
|
airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
|
142
142
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
|
143
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
143
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=60w0FR8iU4O4J-HdE7bMRYD7TXuCO4MgXLah1_4GX2Q,174107
|
144
144
|
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
|
145
145
|
airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
|
146
146
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
@@ -215,17 +215,17 @@ airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=J8Q_iJYhc
|
|
215
215
|
airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRsatRJq3R3BeyRR0wIoK3gcP1gcpVRQ_P5U,464
|
216
216
|
airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py,sha256=5Wl-fqW-pVf_dxJ4yGHMAFfC4JjKHYJhqFJT1xA57F4,4177
|
217
217
|
airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLnrDLxf1PJKdUqvQq2RVnAOAzNSY,379
|
218
|
-
airbyte_cdk/sources/declarative/spec/__init__.py,sha256=
|
219
|
-
airbyte_cdk/sources/declarative/spec/spec.py,sha256=
|
218
|
+
airbyte_cdk/sources/declarative/spec/__init__.py,sha256=9FYO-fVOclrwjAW4qwRTbZRVopTc9rOaauAJfThdNCQ,177
|
219
|
+
airbyte_cdk/sources/declarative/spec/spec.py,sha256=eOdIh_Jzlq0Tbmgrx4TSR-6kBxl7nu9F-I-zHBcS0JI,4734
|
220
220
|
airbyte_cdk/sources/declarative/stream_slicers/__init__.py,sha256=sI9vhc95RwJYOnA0VKjcbtKgFcmAbWjhdWBXFbAijOs,176
|
221
221
|
airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py,sha256=cjKGm4r438dd1GxrFHJ4aYrdzG2bkncnwaWxAwlXR3M,3585
|
222
222
|
airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
|
223
223
|
airbyte_cdk/sources/declarative/transformations/__init__.py,sha256=CPJ8TlMpiUmvG3624VYu_NfTzxwKcfBjM2Q2wJ7fkSA,919
|
224
224
|
airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=Eg1jQtRObgzxbtySTQs5uEZIjEklsoHFxYSPf78x6Ng,5420
|
225
225
|
airbyte_cdk/sources/declarative/transformations/config_transformations/__init__.py,sha256=GaU3ezFa5opeDgdlNohX6TXsWJlOD2jOfJXQWeQCh7E,263
|
226
|
-
airbyte_cdk/sources/declarative/transformations/config_transformations/add_fields.py,sha256=
|
226
|
+
airbyte_cdk/sources/declarative/transformations/config_transformations/add_fields.py,sha256=pP9Er3CI0xbHhQDIDsDSg9TwhuL14wamcMXUY5pdc9g,3902
|
227
227
|
airbyte_cdk/sources/declarative/transformations/config_transformations/config_transformation.py,sha256=vh-NAe6-1H77tYP6MrH4qz_fxCDgoYoi1YlbT1yScrA,629
|
228
|
-
airbyte_cdk/sources/declarative/transformations/config_transformations/remap_field.py,sha256=
|
228
|
+
airbyte_cdk/sources/declarative/transformations/config_transformations/remap_field.py,sha256=5UG48Uvqo5_H0-dZJEMu7ONwrHyeDkJ6TToPz9OE4vU,2360
|
229
229
|
airbyte_cdk/sources/declarative/transformations/config_transformations/remove_fields.py,sha256=mIoGM4fywxdlhdNAWHRfQbzOcP7rCdGEn7Yu8jDN1dY,2316
|
230
230
|
airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py,sha256=DO_zR2TqlvLTRO0c572xrleI4V-1QWVOEhbenGXVMLc,3811
|
231
231
|
airbyte_cdk/sources/declarative/transformations/flatten_fields.py,sha256=yT3owG6rMKaRX-LJ_T-jSTnh1B5NoAHyH4YZN9yOvE8,1758
|
@@ -236,7 +236,7 @@ airbyte_cdk/sources/declarative/transformations/remove_fields.py,sha256=EwUP0SZ2
|
|
236
236
|
airbyte_cdk/sources/declarative/transformations/transformation.py,sha256=4sXtx9cNY2EHUPq-xHvDs8GQEBUy3Eo6TkRLKHPXx68,1161
|
237
237
|
airbyte_cdk/sources/declarative/types.py,sha256=yqx0xlZv_76tkC7fqJKefmvl4GJJ8mXbeddwVV8XRJU,778
|
238
238
|
airbyte_cdk/sources/declarative/validators/__init__.py,sha256=_xccLn4QKQqR3CAwmCVOA7l6QuJd_Isoh6NsR8crM2U,663
|
239
|
-
airbyte_cdk/sources/declarative/validators/dpath_validator.py,sha256=
|
239
|
+
airbyte_cdk/sources/declarative/validators/dpath_validator.py,sha256=BF1KR20RWew_wPV3rDmXPOGO8QWZLZu7u5ubbNO1Hgo,2099
|
240
240
|
airbyte_cdk/sources/declarative/validators/predicate_validator.py,sha256=Q4eVnclk1vYPvVF7XROG4MFEVkPYbYKEF8SUKs7P7-k,582
|
241
241
|
airbyte_cdk/sources/declarative/validators/validate_adheres_to_schema.py,sha256=kjcuKxWMJEzpF4GiESITGMxBAXw6YZCAsgOQMgeBo4g,1085
|
242
242
|
airbyte_cdk/sources/declarative/validators/validation_strategy.py,sha256=LwqUX89cFdHTM1-h6c8vebBA9WC38HYoGBvJfCZHr0g,467
|
@@ -419,9 +419,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
419
419
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
420
420
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
421
421
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
422
|
-
airbyte_cdk-6.
|
423
|
-
airbyte_cdk-6.
|
424
|
-
airbyte_cdk-6.
|
425
|
-
airbyte_cdk-6.
|
426
|
-
airbyte_cdk-6.
|
427
|
-
airbyte_cdk-6.
|
422
|
+
airbyte_cdk-6.49.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
423
|
+
airbyte_cdk-6.49.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
424
|
+
airbyte_cdk-6.49.0.dist-info/METADATA,sha256=2RMSDX4QS6JgauUNK2qFlrr47cmgxL93Dm1sOnhT4bc,6343
|
425
|
+
airbyte_cdk-6.49.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
426
|
+
airbyte_cdk-6.49.0.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
|
427
|
+
airbyte_cdk-6.49.0.dist-info/RECORD,,
|
{airbyte_cdk-6.48.16.post14.dev15122056170.dist-info → airbyte_cdk-6.49.0.dist-info}/LICENSE.txt
RENAMED
File without changes
|
{airbyte_cdk-6.48.16.post14.dev15122056170.dist-info → airbyte_cdk-6.49.0.dist-info}/LICENSE_SHORT
RENAMED
File without changes
|
File without changes
|
File without changes
|