airbyte-cdk 6.48.12.post3.dev15027581459__py3-none-any.whl → 6.48.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +2 -2
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +51 -9
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +0 -6
- airbyte_cdk/sources/declarative/transformations/config_transformations/__init__.py +9 -0
- airbyte_cdk/sources/declarative/transformations/config_transformations/add_fields.py +119 -0
- airbyte_cdk/sources/declarative/transformations/config_transformations/config_transformation.py +23 -0
- airbyte_cdk/sources/declarative/transformations/config_transformations/remap_field.py +66 -0
- airbyte_cdk/sources/declarative/transformations/config_transformations/remove_fields.py +66 -0
- airbyte_cdk/sources/declarative/validators/__init__.py +19 -0
- airbyte_cdk/sources/declarative/validators/dpath_validator.py +59 -0
- airbyte_cdk/sources/declarative/validators/predicate_validator.py +26 -0
- airbyte_cdk/sources/declarative/validators/validate_adheres_to_schema.py +39 -0
- airbyte_cdk/sources/declarative/validators/validation_strategy.py +22 -0
- airbyte_cdk/sources/declarative/validators/validator.py +18 -0
- {airbyte_cdk-6.48.12.post3.dev15027581459.dist-info → airbyte_cdk-6.48.13.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.48.12.post3.dev15027581459.dist-info → airbyte_cdk-6.48.13.dist-info}/RECORD +20 -9
- {airbyte_cdk-6.48.12.post3.dev15027581459.dist-info → airbyte_cdk-6.48.13.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.48.12.post3.dev15027581459.dist-info → airbyte_cdk-6.48.13.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.48.12.post3.dev15027581459.dist-info → airbyte_cdk-6.48.13.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.48.12.post3.dev15027581459.dist-info → airbyte_cdk-6.48.13.dist-info}/entry_points.txt +0 -0
@@ -229,10 +229,10 @@ class ConcurrentPerPartitionCursor(Cursor):
|
|
229
229
|
|
230
230
|
def _throttle_state_message(self) -> Optional[float]:
|
231
231
|
"""
|
232
|
-
Throttles the state message emission to once every
|
232
|
+
Throttles the state message emission to once every 600 seconds.
|
233
233
|
"""
|
234
234
|
current_time = time.time()
|
235
|
-
if current_time - self._last_emission_time <=
|
235
|
+
if current_time - self._last_emission_time <= 600:
|
236
236
|
return None
|
237
237
|
return current_time
|
238
238
|
|
@@ -120,14 +120,6 @@ class ManifestComponentTransformer:
|
|
120
120
|
if found_type:
|
121
121
|
propagated_component["type"] = found_type
|
122
122
|
|
123
|
-
# When there is no resolved type, we're not processing a component (likely a regular object) and don't need to propagate parameters
|
124
|
-
# When the type refers to a json schema, we're not processing a component as well. This check is currently imperfect as there could
|
125
|
-
# be json_schema are not objects but we believe this is not likely in our case because:
|
126
|
-
# * records are Mapping so objects hence SchemaLoader root should be an object
|
127
|
-
# * connection_specification is a Mapping
|
128
|
-
if "type" not in propagated_component or self._is_json_schema_object(propagated_component):
|
129
|
-
return propagated_component
|
130
|
-
|
131
123
|
# Combines parameters defined at the current level with parameters from parent components. Parameters at the current
|
132
124
|
# level take precedence
|
133
125
|
current_parameters = dict(copy.deepcopy(parent_parameters))
|
@@ -138,6 +130,27 @@ class ManifestComponentTransformer:
|
|
138
130
|
else {**current_parameters, **component_parameters}
|
139
131
|
)
|
140
132
|
|
133
|
+
# When there is no resolved type, we're not processing a component (likely a regular object) and don't need to propagate parameters
|
134
|
+
# When the type refers to a json schema, we're not processing a component as well. This check is currently imperfect as there could
|
135
|
+
# be json_schema are not objects but we believe this is not likely in our case because:
|
136
|
+
# * records are Mapping so objects hence SchemaLoader root should be an object
|
137
|
+
# * connection_specification is a Mapping
|
138
|
+
if self._is_json_schema_object(propagated_component):
|
139
|
+
return propagated_component
|
140
|
+
|
141
|
+
# For objects that don't have type check if their object fields have nested components which should have `$parameters` in it.
|
142
|
+
# For example, QueryProperties in requester.request_parameters, etc.
|
143
|
+
# Update propagated_component value with nested components with parent `$parameters` if needed and return propagated_component.
|
144
|
+
if "type" not in propagated_component:
|
145
|
+
if self._has_nested_components(propagated_component):
|
146
|
+
propagated_component = self._process_nested_components(
|
147
|
+
propagated_component,
|
148
|
+
parent_field_identifier,
|
149
|
+
current_parameters,
|
150
|
+
use_parent_parameters,
|
151
|
+
)
|
152
|
+
return propagated_component
|
153
|
+
|
141
154
|
# Parameters should be applied to the current component fields with the existing field taking precedence over parameters if
|
142
155
|
# both exist
|
143
156
|
for parameter_key, parameter_value in current_parameters.items():
|
@@ -181,4 +194,33 @@ class ManifestComponentTransformer:
|
|
181
194
|
|
182
195
|
@staticmethod
|
183
196
|
def _is_json_schema_object(propagated_component: Mapping[str, Any]) -> bool:
|
184
|
-
return propagated_component.get("type") == "object"
|
197
|
+
return propagated_component.get("type") == "object" or propagated_component.get("type") == [
|
198
|
+
"null",
|
199
|
+
"object",
|
200
|
+
]
|
201
|
+
|
202
|
+
@staticmethod
|
203
|
+
def _has_nested_components(propagated_component: Dict[str, Any]) -> bool:
|
204
|
+
for k, v in propagated_component.items():
|
205
|
+
if isinstance(v, dict) and v.get("type"):
|
206
|
+
return True
|
207
|
+
return False
|
208
|
+
|
209
|
+
def _process_nested_components(
|
210
|
+
self,
|
211
|
+
propagated_component: Dict[str, Any],
|
212
|
+
parent_field_identifier: str,
|
213
|
+
current_parameters: Mapping[str, Any],
|
214
|
+
use_parent_parameters: Optional[bool] = None,
|
215
|
+
) -> Dict[str, Any]:
|
216
|
+
for field_name, field_value in propagated_component.items():
|
217
|
+
if isinstance(field_value, dict) and field_value.get("type"):
|
218
|
+
nested_component_with_parameters = self.propagate_types_and_parameters(
|
219
|
+
parent_field_identifier,
|
220
|
+
field_value,
|
221
|
+
current_parameters,
|
222
|
+
use_parent_parameters=use_parent_parameters,
|
223
|
+
)
|
224
|
+
propagated_component[field_name] = nested_component_with_parameters
|
225
|
+
|
226
|
+
return propagated_component
|
@@ -1484,12 +1484,6 @@ class ModelToComponentFactory:
|
|
1484
1484
|
stream_state_migrations=stream_state_migrations,
|
1485
1485
|
)
|
1486
1486
|
)
|
1487
|
-
|
1488
|
-
if not stream_state:
|
1489
|
-
stream_state = self._connector_state_manager.get_stream_state(
|
1490
|
-
stream_name, stream_namespace
|
1491
|
-
)
|
1492
|
-
|
1493
1487
|
stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
|
1494
1488
|
# Per-partition state doesn't make sense for GroupingPartitionRouter, so force the global state
|
1495
1489
|
use_global_cursor = isinstance(
|
@@ -0,0 +1,9 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from .add_fields import ConfigAddFields
|
6
|
+
from .remap_field import ConfigRemapField
|
7
|
+
from .remove_fields import ConfigRemoveFields
|
8
|
+
|
9
|
+
__all__ = ["ConfigRemapField", "ConfigAddFields", "ConfigRemoveFields"]
|
@@ -0,0 +1,119 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from dataclasses import dataclass, field
|
6
|
+
from typing import Any, List, MutableMapping, Optional, Type, Union
|
7
|
+
|
8
|
+
import dpath
|
9
|
+
|
10
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
11
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
12
|
+
from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
|
13
|
+
ConfigTransformation,
|
14
|
+
)
|
15
|
+
from airbyte_cdk.sources.types import FieldPointer
|
16
|
+
|
17
|
+
|
18
|
+
@dataclass(frozen=True)
|
19
|
+
class AddedFieldDefinition:
|
20
|
+
"""Defines the field to add on a config"""
|
21
|
+
|
22
|
+
path: FieldPointer
|
23
|
+
value: Union[InterpolatedString, str]
|
24
|
+
value_type: Optional[Type[Any]] = None
|
25
|
+
|
26
|
+
|
27
|
+
@dataclass(frozen=True)
|
28
|
+
class ParsedAddFieldDefinition:
|
29
|
+
"""Defines the field to add on a config"""
|
30
|
+
|
31
|
+
path: FieldPointer
|
32
|
+
value: InterpolatedString
|
33
|
+
value_type: Optional[Type[Any]] = None
|
34
|
+
|
35
|
+
|
36
|
+
@dataclass
|
37
|
+
class ConfigAddFields(ConfigTransformation):
|
38
|
+
"""
|
39
|
+
Transformation which adds fields to a config. The path of the added field can be nested. Adding nested fields will create all
|
40
|
+
necessary parent objects (like mkdir -p).
|
41
|
+
|
42
|
+
This transformation has access to the config being transformed.
|
43
|
+
|
44
|
+
Examples of instantiating this transformation via YAML:
|
45
|
+
- type: ConfigAddFields
|
46
|
+
fields:
|
47
|
+
# hardcoded constant
|
48
|
+
- path: ["path"]
|
49
|
+
value: "static_value"
|
50
|
+
|
51
|
+
# nested path
|
52
|
+
- path: ["path", "to", "field"]
|
53
|
+
value: "static"
|
54
|
+
|
55
|
+
# from config
|
56
|
+
- path: ["derived_field"]
|
57
|
+
value: "{{ config.original_field }}"
|
58
|
+
|
59
|
+
# by supplying any valid Jinja template directive or expression
|
60
|
+
- path: ["two_times_two"]
|
61
|
+
value: "{{ 2 * 2 }}"
|
62
|
+
|
63
|
+
Attributes:
|
64
|
+
fields (List[AddedFieldDefinition]): A list of transformations (path and corresponding value) that will be added to the config
|
65
|
+
"""
|
66
|
+
|
67
|
+
fields: List[AddedFieldDefinition]
|
68
|
+
condition: str = ""
|
69
|
+
_parsed_fields: List[ParsedAddFieldDefinition] = field(
|
70
|
+
init=False, repr=False, default_factory=list
|
71
|
+
)
|
72
|
+
|
73
|
+
def __post_init__(self) -> None:
|
74
|
+
self._filter_interpolator = InterpolatedBoolean(condition=self.condition, parameters={})
|
75
|
+
|
76
|
+
for add_field in self.fields:
|
77
|
+
if len(add_field.path) < 1:
|
78
|
+
raise ValueError(
|
79
|
+
f"Expected a non-zero-length path for the AddFields transformation {add_field}"
|
80
|
+
)
|
81
|
+
|
82
|
+
if not isinstance(add_field.value, InterpolatedString):
|
83
|
+
if not isinstance(add_field.value, str):
|
84
|
+
raise ValueError(
|
85
|
+
f"Expected a string value for the AddFields transformation: {add_field}"
|
86
|
+
)
|
87
|
+
else:
|
88
|
+
self._parsed_fields.append(
|
89
|
+
ParsedAddFieldDefinition(
|
90
|
+
add_field.path,
|
91
|
+
InterpolatedString.create(add_field.value, parameters={}),
|
92
|
+
value_type=add_field.value_type,
|
93
|
+
)
|
94
|
+
)
|
95
|
+
else:
|
96
|
+
self._parsed_fields.append(
|
97
|
+
ParsedAddFieldDefinition(
|
98
|
+
add_field.path,
|
99
|
+
add_field.value,
|
100
|
+
value_type=add_field.value_type,
|
101
|
+
)
|
102
|
+
)
|
103
|
+
|
104
|
+
def transform(
|
105
|
+
self,
|
106
|
+
config: MutableMapping[str, Any],
|
107
|
+
) -> None:
|
108
|
+
"""
|
109
|
+
Transforms a config by adding fields based on the provided field definitions.
|
110
|
+
|
111
|
+
:param config: The user-provided configuration to be transformed
|
112
|
+
"""
|
113
|
+
for parsed_field in self._parsed_fields:
|
114
|
+
valid_types = (parsed_field.value_type,) if parsed_field.value_type else None
|
115
|
+
value = parsed_field.value.eval(config, valid_types=valid_types)
|
116
|
+
if not self.condition or self._filter_interpolator.eval(
|
117
|
+
config, value=value, path=parsed_field.path
|
118
|
+
):
|
119
|
+
dpath.new(config, parsed_field.path, value)
|
airbyte_cdk/sources/declarative/transformations/config_transformations/config_transformation.py
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from abc import ABC, abstractmethod
|
6
|
+
from typing import Any, MutableMapping
|
7
|
+
|
8
|
+
|
9
|
+
class ConfigTransformation(ABC):
|
10
|
+
"""
|
11
|
+
Implementations of this class define transformations that can be applied to source configurations.
|
12
|
+
"""
|
13
|
+
|
14
|
+
@abstractmethod
|
15
|
+
def transform(
|
16
|
+
self,
|
17
|
+
config: MutableMapping[str, Any],
|
18
|
+
) -> None:
|
19
|
+
"""
|
20
|
+
Transform a configuration by adding, deleting, or mutating fields directly from the config reference passed in argument.
|
21
|
+
|
22
|
+
:param config: The user-provided configuration to be transformed
|
23
|
+
"""
|
@@ -0,0 +1,66 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from dataclasses import dataclass, field
|
6
|
+
from typing import Any, List, Mapping, MutableMapping, Union
|
7
|
+
|
8
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
9
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
|
10
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
11
|
+
from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
|
12
|
+
ConfigTransformation,
|
13
|
+
)
|
14
|
+
|
15
|
+
|
16
|
+
@dataclass
|
17
|
+
class ConfigRemapField(ConfigTransformation):
|
18
|
+
"""
|
19
|
+
Transformation that remaps a field's value to another value based on a static map.
|
20
|
+
"""
|
21
|
+
|
22
|
+
map: Mapping[str, Any]
|
23
|
+
field_path: List[Union[InterpolatedString, str]]
|
24
|
+
config: Mapping[str, Any] = field(default_factory=dict)
|
25
|
+
|
26
|
+
def __post_init__(self) -> None:
|
27
|
+
if not self.field_path:
|
28
|
+
raise Exception("field_path cannot be empty.")
|
29
|
+
self._field_path = [
|
30
|
+
InterpolatedString.create(path, parameters={}) for path in self.field_path
|
31
|
+
]
|
32
|
+
for path_index in range(len(self.field_path)):
|
33
|
+
if isinstance(self.field_path[path_index], str):
|
34
|
+
self._field_path[path_index] = InterpolatedString.create(
|
35
|
+
self.field_path[path_index], parameters={}
|
36
|
+
)
|
37
|
+
self._map = InterpolatedMapping(self.map, parameters={})
|
38
|
+
|
39
|
+
def transform(
|
40
|
+
self,
|
41
|
+
config: MutableMapping[str, Any],
|
42
|
+
) -> None:
|
43
|
+
"""
|
44
|
+
Transforms a config by remapping a field value based on the provided map.
|
45
|
+
If the original value is found in the map, it's replaced with the mapped value.
|
46
|
+
If the value is not in the map, the field remains unchanged.
|
47
|
+
|
48
|
+
:param config: The user-provided configuration to be transformed
|
49
|
+
"""
|
50
|
+
path_components = [path.eval(config) for path in self._field_path]
|
51
|
+
|
52
|
+
current = config
|
53
|
+
for i, component in enumerate(path_components[:-1]):
|
54
|
+
if component not in current:
|
55
|
+
return
|
56
|
+
current = current[component]
|
57
|
+
|
58
|
+
if not isinstance(current, MutableMapping):
|
59
|
+
return
|
60
|
+
|
61
|
+
field_name = path_components[-1]
|
62
|
+
|
63
|
+
mapping = self._map.eval(config=self.config)
|
64
|
+
|
65
|
+
if field_name in current and current[field_name] in mapping:
|
66
|
+
current[field_name] = mapping[current[field_name]]
|
@@ -0,0 +1,66 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
from dataclasses import dataclass
|
5
|
+
from typing import Any, List, MutableMapping
|
6
|
+
|
7
|
+
import dpath
|
8
|
+
import dpath.exceptions
|
9
|
+
|
10
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
11
|
+
from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
|
12
|
+
ConfigTransformation,
|
13
|
+
)
|
14
|
+
from airbyte_cdk.sources.types import FieldPointer
|
15
|
+
|
16
|
+
|
17
|
+
@dataclass
|
18
|
+
class ConfigRemoveFields(ConfigTransformation):
|
19
|
+
"""
|
20
|
+
A transformation which removes fields from a config. The fields removed are designated using FieldPointers.
|
21
|
+
During transformation, if a field or any of its parents does not exist in the config, no error is thrown.
|
22
|
+
|
23
|
+
If an input field pointer references an item in a list (e.g: ["k", 0] in the object {"k": ["a", "b", "c"]}) then
|
24
|
+
the object at that index is set to None rather than being entirely removed from the list.
|
25
|
+
|
26
|
+
It's possible to remove objects nested in lists e.g: removing [".", 0, "k"] from {".": [{"k": "V"}]} results in {".": [{}]}
|
27
|
+
|
28
|
+
Usage syntax:
|
29
|
+
|
30
|
+
```yaml
|
31
|
+
config_transformations:
|
32
|
+
- type: RemoveFields
|
33
|
+
field_pointers:
|
34
|
+
- ["path", "to", "field1"]
|
35
|
+
- ["path2"]
|
36
|
+
condition: "{{ config.some_flag }}" # Optional condition
|
37
|
+
```
|
38
|
+
|
39
|
+
Attributes:
|
40
|
+
field_pointers (List[FieldPointer]): pointers to the fields that should be removed
|
41
|
+
condition (str): Optional condition that determines if the fields should be removed
|
42
|
+
"""
|
43
|
+
|
44
|
+
field_pointers: List[FieldPointer]
|
45
|
+
condition: str = ""
|
46
|
+
|
47
|
+
def __post_init__(self) -> None:
|
48
|
+
self._filter_interpolator = InterpolatedBoolean(condition=self.condition, parameters={})
|
49
|
+
|
50
|
+
def transform(
|
51
|
+
self,
|
52
|
+
config: MutableMapping[str, Any],
|
53
|
+
) -> None:
|
54
|
+
"""
|
55
|
+
Transforms a config by removing fields based on the provided field pointers.
|
56
|
+
|
57
|
+
:param config: The user-provided configuration to be transformed
|
58
|
+
"""
|
59
|
+
if self.condition and not self._filter_interpolator.eval(config):
|
60
|
+
return
|
61
|
+
|
62
|
+
for pointer in self.field_pointers:
|
63
|
+
try:
|
64
|
+
dpath.delete(config, pointer)
|
65
|
+
except dpath.exceptions.PathNotFound:
|
66
|
+
pass
|
@@ -0,0 +1,19 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from airbyte_cdk.sources.declarative.validators.dpath_validator import DpathValidator
|
6
|
+
from airbyte_cdk.sources.declarative.validators.predicate_validator import PredicateValidator
|
7
|
+
from airbyte_cdk.sources.declarative.validators.validate_adheres_to_schema import (
|
8
|
+
ValidateAdheresToSchema,
|
9
|
+
)
|
10
|
+
from airbyte_cdk.sources.declarative.validators.validation_strategy import ValidationStrategy
|
11
|
+
from airbyte_cdk.sources.declarative.validators.validator import Validator
|
12
|
+
|
13
|
+
__all__ = [
|
14
|
+
"Validator",
|
15
|
+
"DpathValidator",
|
16
|
+
"ValidationStrategy",
|
17
|
+
"ValidateAdheresToSchema",
|
18
|
+
"PredicateValidator",
|
19
|
+
]
|
@@ -0,0 +1,59 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from dataclasses import dataclass
|
6
|
+
from typing import Any, List, Union
|
7
|
+
|
8
|
+
import dpath.util
|
9
|
+
|
10
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
11
|
+
from airbyte_cdk.sources.declarative.validators.validation_strategy import ValidationStrategy
|
12
|
+
from airbyte_cdk.sources.declarative.validators.validator import Validator
|
13
|
+
|
14
|
+
|
15
|
+
@dataclass
|
16
|
+
class DpathValidator(Validator):
|
17
|
+
"""
|
18
|
+
Validator that extracts a value at a specific path in the input data
|
19
|
+
and applies a validation strategy to it.
|
20
|
+
"""
|
21
|
+
|
22
|
+
field_path: List[Union[InterpolatedString, str]]
|
23
|
+
strategy: ValidationStrategy
|
24
|
+
|
25
|
+
def __post_init__(self) -> None:
|
26
|
+
self._field_path = [
|
27
|
+
InterpolatedString.create(path, parameters={}) for path in self.field_path
|
28
|
+
]
|
29
|
+
for path_index in range(len(self.field_path)):
|
30
|
+
if isinstance(self.field_path[path_index], str):
|
31
|
+
self._field_path[path_index] = InterpolatedString.create(
|
32
|
+
self.field_path[path_index], parameters={}
|
33
|
+
)
|
34
|
+
|
35
|
+
def validate(self, input_data: dict[str, Any]) -> None:
|
36
|
+
"""
|
37
|
+
Extracts the value at the specified path and applies the validation strategy.
|
38
|
+
|
39
|
+
:param input_data: Dictionary containing the data to validate
|
40
|
+
:raises ValueError: If the path doesn't exist or validation fails
|
41
|
+
"""
|
42
|
+
path = [path.eval({}) for path in self._field_path]
|
43
|
+
|
44
|
+
if len(path) == 0:
|
45
|
+
raise ValueError("Field path is empty")
|
46
|
+
|
47
|
+
if "*" in path:
|
48
|
+
try:
|
49
|
+
values = dpath.values(input_data, path)
|
50
|
+
for value in values:
|
51
|
+
self.strategy.validate(value)
|
52
|
+
except KeyError as e:
|
53
|
+
raise ValueError(f"Error validating path '{self.field_path}': {e}")
|
54
|
+
else:
|
55
|
+
try:
|
56
|
+
value = dpath.get(input_data, path)
|
57
|
+
self.strategy.validate(value)
|
58
|
+
except KeyError as e:
|
59
|
+
raise ValueError(f"Error validating path '{self.field_path}': {e}")
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from dataclasses import dataclass
|
6
|
+
from typing import Any
|
7
|
+
|
8
|
+
from airbyte_cdk.sources.declarative.validators.validation_strategy import ValidationStrategy
|
9
|
+
|
10
|
+
|
11
|
+
@dataclass
|
12
|
+
class PredicateValidator:
|
13
|
+
"""
|
14
|
+
Validator that applies a validation strategy to a value.
|
15
|
+
"""
|
16
|
+
|
17
|
+
value: Any
|
18
|
+
strategy: ValidationStrategy
|
19
|
+
|
20
|
+
def validate(self) -> None:
|
21
|
+
"""
|
22
|
+
Applies the validation strategy to the value.
|
23
|
+
|
24
|
+
:raises ValueError: If validation fails
|
25
|
+
"""
|
26
|
+
self.strategy.validate(self.value)
|
@@ -0,0 +1,39 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
import json
|
6
|
+
from dataclasses import dataclass
|
7
|
+
from typing import Any, Mapping
|
8
|
+
|
9
|
+
import jsonschema
|
10
|
+
|
11
|
+
from airbyte_cdk.sources.declarative.validators.validation_strategy import ValidationStrategy
|
12
|
+
|
13
|
+
|
14
|
+
@dataclass
|
15
|
+
class ValidateAdheresToSchema(ValidationStrategy):
|
16
|
+
"""
|
17
|
+
Validates that a value adheres to a specified JSON schema.
|
18
|
+
"""
|
19
|
+
|
20
|
+
schema: Mapping[str, Any]
|
21
|
+
|
22
|
+
def validate(self, value: Any) -> None:
|
23
|
+
"""
|
24
|
+
Validates the value against the JSON schema.
|
25
|
+
|
26
|
+
:param value: The value to validate
|
27
|
+
:raises ValueError: If the value does not adhere to the schema
|
28
|
+
"""
|
29
|
+
|
30
|
+
if isinstance(value, str):
|
31
|
+
try:
|
32
|
+
value = json.loads(value)
|
33
|
+
except json.JSONDecodeError as e:
|
34
|
+
raise ValueError(f"Invalid JSON string: {value}") from e
|
35
|
+
|
36
|
+
try:
|
37
|
+
jsonschema.validate(instance=value, schema=self.schema)
|
38
|
+
except jsonschema.ValidationError as e:
|
39
|
+
raise ValueError(f"JSON schema validation error: {e.message}")
|
@@ -0,0 +1,22 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from abc import ABC, abstractmethod
|
6
|
+
from typing import Any
|
7
|
+
|
8
|
+
|
9
|
+
class ValidationStrategy(ABC):
|
10
|
+
"""
|
11
|
+
Base class for validation strategies.
|
12
|
+
"""
|
13
|
+
|
14
|
+
@abstractmethod
|
15
|
+
def validate(self, value: Any) -> None:
|
16
|
+
"""
|
17
|
+
Validates a value according to a specific strategy.
|
18
|
+
|
19
|
+
:param value: The value to validate
|
20
|
+
:raises ValueError: If validation fails
|
21
|
+
"""
|
22
|
+
pass
|
@@ -0,0 +1,18 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from abc import ABC, abstractmethod
|
6
|
+
from typing import Any
|
7
|
+
|
8
|
+
|
9
|
+
class Validator(ABC):
|
10
|
+
@abstractmethod
|
11
|
+
def validate(self, input_data: Any) -> None:
|
12
|
+
"""
|
13
|
+
Validates the input data.
|
14
|
+
|
15
|
+
:param input_data: The data to validate
|
16
|
+
:raises ValueError: If validation fails
|
17
|
+
"""
|
18
|
+
pass
|
@@ -111,7 +111,7 @@ airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=vCpwX1PVRFP
|
|
111
111
|
airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=WJyA2OYIEgFpVP5Y3o0tIj69AV6IKkn9B16MeXaEItI,6513
|
112
112
|
airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
|
113
113
|
airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
|
114
|
-
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=
|
114
|
+
airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=8zxfM1aXSwdfTbpL9PHwgg43ZRbNUQRcX038D7YTI_0,22768
|
115
115
|
airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
|
116
116
|
airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
|
117
117
|
airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
|
@@ -137,10 +137,10 @@ airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=Fj
|
|
137
137
|
airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
138
138
|
airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
|
139
139
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9Rbu5OexXSDN9QWDo8YU4tT9M2LDVOgGA,802
|
140
|
-
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=
|
140
|
+
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=2UdpCz3yi7ISZTyqkQXSSy3dMxeyOWqV7OlAS5b9GVg,11568
|
141
141
|
airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
|
142
142
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
|
143
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
143
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=Ai-3RfSoBa3oNc34KNlUvp7KbbQuuqhtGwGOxPf8fMc,167671
|
144
144
|
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
|
145
145
|
airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
|
146
146
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
@@ -222,6 +222,11 @@ airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.p
|
|
222
222
|
airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py,sha256=SOkIPBi2Wu7yxIvA15yFzUAB95a3IzA8LPq5DEqHQQc,725
|
223
223
|
airbyte_cdk/sources/declarative/transformations/__init__.py,sha256=CPJ8TlMpiUmvG3624VYu_NfTzxwKcfBjM2Q2wJ7fkSA,919
|
224
224
|
airbyte_cdk/sources/declarative/transformations/add_fields.py,sha256=Eg1jQtRObgzxbtySTQs5uEZIjEklsoHFxYSPf78x6Ng,5420
|
225
|
+
airbyte_cdk/sources/declarative/transformations/config_transformations/__init__.py,sha256=GaU3ezFa5opeDgdlNohX6TXsWJlOD2jOfJXQWeQCh7E,263
|
226
|
+
airbyte_cdk/sources/declarative/transformations/config_transformations/add_fields.py,sha256=SwZ32to9j6kJZXjWVDU2vxQ4UklwbekfOL7O_oTDtMU,4144
|
227
|
+
airbyte_cdk/sources/declarative/transformations/config_transformations/config_transformation.py,sha256=vh-NAe6-1H77tYP6MrH4qz_fxCDgoYoi1YlbT1yScrA,629
|
228
|
+
airbyte_cdk/sources/declarative/transformations/config_transformations/remap_field.py,sha256=k2FLHaOVlUDYfns5pPp2hIG6cP8hLRozpHg_fTrSYI0,2493
|
229
|
+
airbyte_cdk/sources/declarative/transformations/config_transformations/remove_fields.py,sha256=mIoGM4fywxdlhdNAWHRfQbzOcP7rCdGEn7Yu8jDN1dY,2316
|
225
230
|
airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py,sha256=DO_zR2TqlvLTRO0c572xrleI4V-1QWVOEhbenGXVMLc,3811
|
226
231
|
airbyte_cdk/sources/declarative/transformations/flatten_fields.py,sha256=yT3owG6rMKaRX-LJ_T-jSTnh1B5NoAHyH4YZN9yOvE8,1758
|
227
232
|
airbyte_cdk/sources/declarative/transformations/keys_replace_transformation.py,sha256=vbIn6ump-Ut6g20yMub7PFoPBhOKVtrHSAUdcOUdLfw,1999
|
@@ -230,6 +235,12 @@ airbyte_cdk/sources/declarative/transformations/keys_to_snake_transformation.py,
|
|
230
235
|
airbyte_cdk/sources/declarative/transformations/remove_fields.py,sha256=EwUP0SZ2p4GRJ6Q8CUzlz9dcUeEidEFDlI2IBye2tlc,2745
|
231
236
|
airbyte_cdk/sources/declarative/transformations/transformation.py,sha256=4sXtx9cNY2EHUPq-xHvDs8GQEBUy3Eo6TkRLKHPXx68,1161
|
232
237
|
airbyte_cdk/sources/declarative/types.py,sha256=yqx0xlZv_76tkC7fqJKefmvl4GJJ8mXbeddwVV8XRJU,778
|
238
|
+
airbyte_cdk/sources/declarative/validators/__init__.py,sha256=_xccLn4QKQqR3CAwmCVOA7l6QuJd_Isoh6NsR8crM2U,663
|
239
|
+
airbyte_cdk/sources/declarative/validators/dpath_validator.py,sha256=DzyHYVOm94yvqB5if2JbFbP0SNGn3QI45DR6biU1zrY,2126
|
240
|
+
airbyte_cdk/sources/declarative/validators/predicate_validator.py,sha256=Q4eVnclk1vYPvVF7XROG4MFEVkPYbYKEF8SUKs7P7-k,582
|
241
|
+
airbyte_cdk/sources/declarative/validators/validate_adheres_to_schema.py,sha256=kjcuKxWMJEzpF4GiESITGMxBAXw6YZCAsgOQMgeBo4g,1085
|
242
|
+
airbyte_cdk/sources/declarative/validators/validation_strategy.py,sha256=LwqUX89cFdHTM1-h6c8vebBA9WC38HYoGBvJfCZHr0g,467
|
243
|
+
airbyte_cdk/sources/declarative/validators/validator.py,sha256=MAwo8OievUsuzBuPxI9pbPu87yq0tJZkGbydcrHZyQc,382
|
233
244
|
airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256=nJCZkzLGP-dwvfwKsl4VqQFZQdhx6fiGCRez1gma0wE,2714
|
234
245
|
airbyte_cdk/sources/file_based/README.md,sha256=iMqww4VZ882jfNQIdljjDgqreKs-mkdtSrRKA94iX6A,11085
|
235
246
|
airbyte_cdk/sources/file_based/__init__.py,sha256=EaxHv_9ot-eRlUCR47ZMZ0IOtB-n0HH24om7Bfn-uuQ,868
|
@@ -408,9 +419,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
408
419
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
409
420
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
410
421
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
411
|
-
airbyte_cdk-6.48.
|
412
|
-
airbyte_cdk-6.48.
|
413
|
-
airbyte_cdk-6.48.
|
414
|
-
airbyte_cdk-6.48.
|
415
|
-
airbyte_cdk-6.48.
|
416
|
-
airbyte_cdk-6.48.
|
422
|
+
airbyte_cdk-6.48.13.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
423
|
+
airbyte_cdk-6.48.13.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
424
|
+
airbyte_cdk-6.48.13.dist-info/METADATA,sha256=GdzqhkyDm8Slz4AKF7xh6l6TPhz8xKHsFG8qrb_AtA4,6344
|
425
|
+
airbyte_cdk-6.48.13.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
426
|
+
airbyte_cdk-6.48.13.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
|
427
|
+
airbyte_cdk-6.48.13.dist-info/RECORD,,
|
{airbyte_cdk-6.48.12.post3.dev15027581459.dist-info → airbyte_cdk-6.48.13.dist-info}/LICENSE.txt
RENAMED
File without changes
|
{airbyte_cdk-6.48.12.post3.dev15027581459.dist-info → airbyte_cdk-6.48.13.dist-info}/LICENSE_SHORT
RENAMED
File without changes
|
File without changes
|
File without changes
|